Evaluate Models
Author: Aaron S Coyner, PhD
Last update: 2021-09-02
Setup
library(tidyverse)
library(tools)
library(caret)
library(ROCR)
library(caret)compute_auroc <- function(predictions, labels, title) {
pred <- prediction(predictions, labels)
auroc <- performance(pred, measure = 'auc')
auroc <- auroc@y.values[[1]]
perf <- performance(pred, 'tpr', 'fpr')
plot(perf,
main = title,
xlim=c(0,1),
ylim=c(0,1),
colorize = FALSE)
abline(a = 0,
b = 1,
col = 'red',
lty = 2)
text(0.8,
0.25,
paste('AUROC:', sprintf('%.3f', round(auroc, 3))))
}compute_aupr <- function(predictions, labels, title) {
n_black <- length(labels[labels == 1])
n_total <- length(labels)
pred <- prediction(predictions, labels)
aupr <- performance(pred, measure = 'aucpr')
aupr <- aupr@y.values[[1]]
perf <- performance(pred, 'prec', 'rec')
plot(perf,
main = title,
xlim=c(0,1),
ylim=c(0,1),
colorize = FALSE)
abline(a = n_black/n_total,
b = 0,
col = 'red',
lty = 2)
text(0.2,
0.2,
paste('Null AUPR:', sprintf('%.3f', round(n_black / n_total, 3))))
text(0.8,
0.2,
paste('AUPR:', sprintf('%.3f', round(aupr, 3))))
}confusion_matrix <- function(df, column, threshold=0.5) {
preds <- as.factor(if_else(df[column] >= threshold, 1, 0))
confusionMatrix(preds, df$label, positive = '1')
}Load data
test_data <- read_csv('./out/datasets/test_data.csv') %>%
select(subject_id, image_id)
── Column specification ──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────
cols(
subject_id = col_character(),
race = col_character(),
variable = col_character(),
value = col_character(),
image_id = col_double(),
fundus_location = col_character(),
segmentation_location = col_character()
)image_level <- read_csv('./out/probabilities/retcam_filtered_0.csv', col_types = cols()) %>%
mutate(label = as.factor(if_else(str_detect(img_loc, 'black'), 1, 0)),
image_id = as.numeric(file_path_sans_ext(basename(img_loc)))) %>%
select(img_loc, image_id, label, retcam = probability) %>%
bind_cols(select(read_csv('./out/probabilities/retcam_filtered_0_random.csv', col_types = cols()), retcam_random = probability)) %>%
bind_cols(select(read_csv('./out/probabilities/segmentations_filtered_0_random.csv', col_types = cols()), filter_0_random = probability)) %>%
bind_cols(select(read_csv('./out/probabilities/segmentations_filtered_0.csv', col_types = cols()), filter_0 = probability)) %>%
bind_cols(select(read_csv('./out/probabilities/segmentations_filtered_0_binarized.csv', col_types = cols()), filter_0_binarized = probability)) %>%
bind_cols(select(read_csv('./out/probabilities/segmentations_filtered_0_skeletonized.csv', col_types = cols()), filter_0_skeletonized = probability)) %>%
bind_cols(select(read_csv('./out/probabilities/segmentations_filtered_50.csv', col_types = cols()), filter_50 = probability)) %>%
bind_cols(select(read_csv('./out/probabilities/segmentations_filtered_50_binarized.csv', col_types = cols()), filter_50_binarized = probability)) %>%
bind_cols(select(read_csv('./out/probabilities/segmentations_filtered_50_skeletonized.csv', col_types = cols()), filter_50_skeletonized = probability)) %>%
bind_cols(select(read_csv('./out/probabilities/segmentations_filtered_100.csv', col_types = cols()), filter_100 = probability)) %>%
bind_cols(select(read_csv('./out/probabilities/segmentations_filtered_100_binarized.csv', col_types = cols()), filter_100_binarized = probability)) %>%
bind_cols(select(read_csv('./out/probabilities/segmentations_filtered_100_skeletonized.csv', col_types = cols()), filter_100_skeletonized = probability)) %>%
bind_cols(select(read_csv('./out/probabilities/segmentations_filtered_150.csv', col_types = cols()), filter_150 = probability)) %>%
bind_cols(select(read_csv('./out/probabilities/segmentations_filtered_150_binarized.csv', col_types = cols()), filter_150_binarized = probability)) %>%
bind_cols(select(read_csv('./out/probabilities/segmentations_filtered_150_skeletonized.csv', col_types = cols()), filter_150_skeletonized = probability)) %>%
bind_cols(select(read_csv('./out/probabilities/segmentations_filtered_200.csv', col_types = cols()), filter_200 = probability)) %>%
bind_cols(select(read_csv('./out/probabilities/segmentations_filtered_200_binarized.csv', col_types = cols()), filter_200_binarized = probability)) %>%
bind_cols(select(read_csv('./out/probabilities/segmentations_filtered_200_skeletonized.csv', col_types = cols()), filter_200_skeletonized = probability)) %>%
bind_cols(select(read_csv('./out/probabilities/segmentations_filtered_210.csv', col_types = cols()), filter_210 = probability)) %>%
bind_cols(select(read_csv('./out/probabilities/segmentations_filtered_210_binarized.csv', col_types = cols()), filter_210_binarized = probability)) %>%
bind_cols(select(read_csv('./out/probabilities/segmentations_filtered_210_skeletonized.csv', col_types = cols()), filter_210_skeletonized = probability)) %>%
bind_cols(select(read_csv('./out/probabilities/segmentations_filtered_220.csv', col_types = cols()), filter_220 = probability)) %>%
bind_cols(select(read_csv('./out/probabilities/segmentations_filtered_220_binarized.csv', col_types = cols()), filter_220_binarized = probability)) %>%
bind_cols(select(read_csv('./out/probabilities/segmentations_filtered_220_skeletonized.csv', col_types = cols()), filter_220_skeletonized = probability)) %>%
bind_cols(select(read_csv('./out/probabilities/segmentations_filtered_230.csv', col_types = cols()), filter_230 = probability)) %>%
bind_cols(select(read_csv('./out/probabilities/segmentations_filtered_230_binarized.csv', col_types = cols()), filter_230_binarized = probability)) %>%
bind_cols(select(read_csv('./out/probabilities/segmentations_filtered_230_skeletonized.csv', col_types = cols()), filter_230_skeletonized = probability)) %>%
bind_cols(select(read_csv('./out/probabilities/segmentations_filtered_240.csv', col_types = cols()), filter_240 = probability)) %>%
bind_cols(select(read_csv('./out/probabilities/segmentations_filtered_240_binarized.csv', col_types = cols()), filter_240_binarized = probability)) %>%
bind_cols(select(read_csv('./out/probabilities/segmentations_filtered_240_skeletonized.csv', col_types = cols()), filter_240_skeletonized = probability)) %>%
bind_cols(select(read_csv('./out/probabilities/segmentations_filtered_250.csv', col_types = cols()), filter_250 = probability)) %>%
bind_cols(select(read_csv('./out/probabilities/segmentations_filtered_250_binarized.csv', col_types = cols()), filter_250_binarized = probability)) %>%
bind_cols(select(read_csv('./out/probabilities/segmentations_filtered_250_skeletonized.csv', col_types = cols()), filter_250_skeletonized = probability)) %>%
bind_cols(select(read_csv('./out/probabilities/segmentations_filtered_257.csv', col_types = cols()), filter_257 = probability)) %>%
bind_cols(select(read_csv('./out/probabilities/segmentations_filtered_257_binarized.csv', col_types = cols()), filter_257_binarized = probability)) %>%
bind_cols(select(read_csv('./out/probabilities/segmentations_filtered_257_skeletonized.csv', col_types = cols()), filter_257_skeletonized = probability)) %>%
bind_cols(select(read_csv('./out/probabilities/segmentations_filtered_0_10.csv', col_types = cols()), filter_10 = probability)) %>%
bind_cols(select(read_csv('./out/probabilities/segmentations_filtered_0_10_binarized.csv', col_types = cols()), filter_10_binarized = probability)) %>%
bind_cols(select(read_csv('./out/probabilities/segmentations_filtered_0_10_skeletonized.csv', col_types = cols()), filter_10_skeletonized = probability)) %>%
bind_cols(select(read_csv('./out/probabilities/segmentations_filtered_75_150.csv', col_types = cols()), filter_75 = probability)) %>%
bind_cols(select(read_csv('./out/probabilities/segmentations_filtered_75_150_binarized.csv', col_types = cols()), filter_75_binarized = probability)) %>%
bind_cols(select(read_csv('./out/probabilities/segmentations_filtered_75_150_skeletonized.csv', col_types = cols()), filter_75_skeletonized = probability)) %>%
inner_join(test_data, by = 'image_id') %>%
select(subject_id, everything(), -img_loc)
subject_level <- image_level %>%
group_by(subject_id) %>%
mutate(across(c(-image_id, -label), median)) %>%
ungroup() %>%
distinct(subject_id, .keep_all = TRUE)Image-level Analysis
RetCam Images
compute_aupr(image_level$retcam, image_level$label, 'PR: Raw RetCam Images')
compute_auroc(image_level$retcam, image_level$label, 'ROC: Raw RetCam Images')
confusion_matrix(image_level, 'retcam')Confusion Matrix and Statistics
Reference
Prediction 0 1
0 819 10
1 8 513
Accuracy : 0.9867
95% CI : (0.979, 0.9921)
No Information Rate : 0.6126
P-Value [Acc > NIR] : <2e-16
Kappa : 0.9719
Mcnemar's Test P-Value : 0.8137
Sensitivity : 0.9809
Specificity : 0.9903
Pos Pred Value : 0.9846
Neg Pred Value : 0.9879
Prevalence : 0.3874
Detection Rate : 0.3800
Detection Prevalence : 0.3859
Balanced Accuracy : 0.9856
'Positive' Class : 1
compute_aupr(image_level$retcam_random, image_level$label, 'PR: Raw RetCam Images - Shuffled Labels')
compute_auroc(image_level$retcam_random, image_level$label, 'ROC: Raw RetCam Images - Shuffled Labels')
confusion_matrix(image_level, 'retcam_random')Confusion Matrix and Statistics
Reference
Prediction 0 1
0 444 113
1 383 410
Accuracy : 0.6326
95% CI : (0.6062, 0.6584)
No Information Rate : 0.6126
P-Value [Acc > NIR] : 0.06902
Kappa : 0.293
Mcnemar's Test P-Value : < 2e-16
Sensitivity : 0.7839
Specificity : 0.5369
Pos Pred Value : 0.5170
Neg Pred Value : 0.7971
Prevalence : 0.3874
Detection Rate : 0.3037
Detection Prevalence : 0.5874
Balanced Accuracy : 0.6604
'Positive' Class : 1
Segmentations: zero all pixels < 0
compute_aupr(image_level$filter_0, image_level$label, 'PR: Segmentations: zero all pixels < 0')
compute_auroc(image_level$filter_0, image_level$label, 'ROC: Segmentations: zero all pixels < 0')
confusion_matrix(image_level, 'filter_0')Confusion Matrix and Statistics
Reference
Prediction 0 1
0 758 74
1 69 449
Accuracy : 0.8941
95% CI : (0.8764, 0.91)
No Information Rate : 0.6126
P-Value [Acc > NIR] : <2e-16
Kappa : 0.7764
Mcnemar's Test P-Value : 0.738
Sensitivity : 0.8585
Specificity : 0.9166
Pos Pred Value : 0.8668
Neg Pred Value : 0.9111
Prevalence : 0.3874
Detection Rate : 0.3326
Detection Prevalence : 0.3837
Balanced Accuracy : 0.8875
'Positive' Class : 1
compute_aupr(image_level$filter_0_binarized, image_level$label, 'PR: Segmentations: zero all pixels < 0 and binarize')
compute_auroc(image_level$filter_0_binarized, image_level$label, 'ROC: Segmentations: zero all pixels < 0 and binarize')
confusion_matrix(image_level, 'filter_0_binarized')Confusion Matrix and Statistics
Reference
Prediction 0 1
0 733 31
1 94 492
Accuracy : 0.9074
95% CI : (0.8907, 0.9223)
No Information Rate : 0.6126
P-Value [Acc > NIR] : < 2.2e-16
Kappa : 0.8091
Mcnemar's Test P-Value : 2.932e-08
Sensitivity : 0.9407
Specificity : 0.8863
Pos Pred Value : 0.8396
Neg Pred Value : 0.9594
Prevalence : 0.3874
Detection Rate : 0.3644
Detection Prevalence : 0.4341
Balanced Accuracy : 0.9135
'Positive' Class : 1
compute_aupr(image_level$filter_0_skeletonized, image_level$label, 'PR: Segmentations: zero all pixels < 0 and skeletonize')
compute_auroc(image_level$filter_0_skeletonized, image_level$label, 'ROC: Segmentations: zero all pixels < 0 and skeletonize')
confusion_matrix(image_level, 'filter_0_skeletonized')Confusion Matrix and Statistics
Reference
Prediction 0 1
0 691 38
1 136 485
Accuracy : 0.8711
95% CI : (0.8521, 0.8885)
No Information Rate : 0.6126
P-Value [Acc > NIR] : < 2.2e-16
Kappa : 0.7375
Mcnemar's Test P-Value : 1.93e-13
Sensitivity : 0.9273
Specificity : 0.8356
Pos Pred Value : 0.7810
Neg Pred Value : 0.9479
Prevalence : 0.3874
Detection Rate : 0.3593
Detection Prevalence : 0.4600
Balanced Accuracy : 0.8814
'Positive' Class : 1
compute_aupr(image_level$filter_0_random, image_level$label, 'PR: Segmentations: zero all pixels < 0 - Shuffled Labels')
compute_auroc(image_level$filter_0_random, image_level$label, 'ROC: Segmentations: zero all pixels < 0 - Shuffled Labels')
confusion_matrix(image_level, 'filter_0_random')Confusion Matrix and Statistics
Reference
Prediction 0 1
0 703 418
1 124 105
Accuracy : 0.5985
95% CI : (0.5718, 0.6248)
No Information Rate : 0.6126
P-Value [Acc > NIR] : 0.8619
Kappa : 0.0567
Mcnemar's Test P-Value : <2e-16
Sensitivity : 0.20076
Specificity : 0.85006
Pos Pred Value : 0.45852
Neg Pred Value : 0.62712
Prevalence : 0.38741
Detection Rate : 0.07778
Detection Prevalence : 0.16963
Balanced Accuracy : 0.52541
'Positive' Class : 1
Segmentations: zero all pixels < 50
compute_aupr(image_level$filter_50, image_level$label, 'PR: Segmentations: zero all pixels < 50')
compute_auroc(image_level$filter_50, image_level$label, 'ROC: Segmentations: zero all pixels < 50')
confusion_matrix(image_level, 'filter_50')Confusion Matrix and Statistics
Reference
Prediction 0 1
0 588 63
1 239 460
Accuracy : 0.7763
95% CI : (0.7531, 0.7983)
No Information Rate : 0.6126
P-Value [Acc > NIR] : < 2.2e-16
Kappa : 0.5561
Mcnemar's Test P-Value : < 2.2e-16
Sensitivity : 0.8795
Specificity : 0.7110
Pos Pred Value : 0.6581
Neg Pred Value : 0.9032
Prevalence : 0.3874
Detection Rate : 0.3407
Detection Prevalence : 0.5178
Balanced Accuracy : 0.7953
'Positive' Class : 1
compute_aupr(image_level$filter_50_binarized, image_level$label, 'PR: Segmentations: zero all pixels < 50 and binarize')
compute_auroc(image_level$filter_50_binarized, image_level$label, 'ROC: Segmentations: zero all pixels < 50 and binarize')
confusion_matrix(image_level, 'filter_0_binarized')Confusion Matrix and Statistics
Reference
Prediction 0 1
0 733 31
1 94 492
Accuracy : 0.9074
95% CI : (0.8907, 0.9223)
No Information Rate : 0.6126
P-Value [Acc > NIR] : < 2.2e-16
Kappa : 0.8091
Mcnemar's Test P-Value : 2.932e-08
Sensitivity : 0.9407
Specificity : 0.8863
Pos Pred Value : 0.8396
Neg Pred Value : 0.9594
Prevalence : 0.3874
Detection Rate : 0.3644
Detection Prevalence : 0.4341
Balanced Accuracy : 0.9135
'Positive' Class : 1
compute_aupr(image_level$filter_50_skeletonized, image_level$label, 'PR: Segmentations: zero all pixels < 50 and skeletonize')
compute_auroc(image_level$filter_50_skeletonized, image_level$label, 'ROC: Segmentations: zero all pixels < 50 and skeletonize')
confusion_matrix(image_level, 'filter_50_skeletonized')Confusion Matrix and Statistics
Reference
Prediction 0 1
0 408 20
1 419 503
Accuracy : 0.6748
95% CI : (0.6491, 0.6998)
No Information Rate : 0.6126
P-Value [Acc > NIR] : 1.202e-06
Kappa : 0.3991
Mcnemar's Test P-Value : < 2.2e-16
Sensitivity : 0.9618
Specificity : 0.4933
Pos Pred Value : 0.5456
Neg Pred Value : 0.9533
Prevalence : 0.3874
Detection Rate : 0.3726
Detection Prevalence : 0.6830
Balanced Accuracy : 0.7276
'Positive' Class : 1
Segmentations: zero all pixels < 100
compute_aupr(image_level$filter_100, image_level$label, 'PR: Segmentations: zero all pixels < 100')
compute_auroc(image_level$filter_100, image_level$label, 'ROC: Segmentations: zero all pixels < 100')
confusion_matrix(image_level, 'filter_100')Confusion Matrix and Statistics
Reference
Prediction 0 1
0 736 210
1 91 313
Accuracy : 0.777
95% CI : (0.7539, 0.799)
No Information Rate : 0.6126
P-Value [Acc > NIR] : < 2.2e-16
Kappa : 0.5098
Mcnemar's Test P-Value : 1.036e-11
Sensitivity : 0.5985
Specificity : 0.8900
Pos Pred Value : 0.7748
Neg Pred Value : 0.7780
Prevalence : 0.3874
Detection Rate : 0.2319
Detection Prevalence : 0.2993
Balanced Accuracy : 0.7442
'Positive' Class : 1
compute_aupr(image_level$filter_100_binarized, image_level$label, 'PR: Segmentations: zero all pixels < 100 and binarize')
compute_auroc(image_level$filter_100_binarized, image_level$label, 'ROC: Segmentations: zero all pixels < 100 and binarize')
confusion_matrix(image_level, 'filter_100_binarized')Confusion Matrix and Statistics
Reference
Prediction 0 1
0 669 142
1 158 381
Accuracy : 0.7778
95% CI : (0.7546, 0.7997)
No Information Rate : 0.6126
P-Value [Acc > NIR] : <2e-16
Kappa : 0.5344
Mcnemar's Test P-Value : 0.3865
Sensitivity : 0.7285
Specificity : 0.8089
Pos Pred Value : 0.7069
Neg Pred Value : 0.8249
Prevalence : 0.3874
Detection Rate : 0.2822
Detection Prevalence : 0.3993
Balanced Accuracy : 0.7687
'Positive' Class : 1
compute_aupr(image_level$filter_100_skeletonized, image_level$label, 'PR: Segmentations: zero all pixels < 100 and skeletonize')
compute_auroc(image_level$filter_100_skeletonized, image_level$label, 'ROC: Segmentations: zero all pixels < 100 and skeletonize')
confusion_matrix(image_level, 'filter_100_skeletonized')Confusion Matrix and Statistics
Reference
Prediction 0 1
0 632 144
1 195 379
Accuracy : 0.7489
95% CI : (0.7249, 0.7718)
No Information Rate : 0.6126
P-Value [Acc > NIR] : < 2.2e-16
Kappa : 0.4803
Mcnemar's Test P-Value : 0.006615
Sensitivity : 0.7247
Specificity : 0.7642
Pos Pred Value : 0.6603
Neg Pred Value : 0.8144
Prevalence : 0.3874
Detection Rate : 0.2807
Detection Prevalence : 0.4252
Balanced Accuracy : 0.7444
'Positive' Class : 1
Segmentations: zero all pixels < 150
compute_aupr(image_level$filter_150, image_level$label, 'PR: Segmentations: zero all pixels < 150')
compute_auroc(image_level$filter_150, image_level$label, 'ROC: Segmentations: zero all pixels < 150')
confusion_matrix(image_level, 'filter_150')Confusion Matrix and Statistics
Reference
Prediction 0 1
0 625 118
1 202 405
Accuracy : 0.763
95% CI : (0.7393, 0.7854)
No Information Rate : 0.6126
P-Value [Acc > NIR] : < 2.2e-16
Kappa : 0.5149
Mcnemar's Test P-Value : 3.487e-06
Sensitivity : 0.7744
Specificity : 0.7557
Pos Pred Value : 0.6672
Neg Pred Value : 0.8412
Prevalence : 0.3874
Detection Rate : 0.3000
Detection Prevalence : 0.4496
Balanced Accuracy : 0.7651
'Positive' Class : 1
compute_aupr(image_level$filter_150_binarized, image_level$label, 'PR: Segmentations: zero all pixels < 150 and binarize')
compute_auroc(image_level$filter_150_binarized, image_level$label, 'ROC: Segmentations: zero all pixels < 150 and binarize')
confusion_matrix(image_level, 'filter_150_binarized')Confusion Matrix and Statistics
Reference
Prediction 0 1
0 702 167
1 125 356
Accuracy : 0.7837
95% CI : (0.7608, 0.8054)
No Information Rate : 0.6126
P-Value [Acc > NIR] : < 2e-16
Kappa : 0.5375
Mcnemar's Test P-Value : 0.01642
Sensitivity : 0.6807
Specificity : 0.8489
Pos Pred Value : 0.7401
Neg Pred Value : 0.8078
Prevalence : 0.3874
Detection Rate : 0.2637
Detection Prevalence : 0.3563
Balanced Accuracy : 0.7648
'Positive' Class : 1
compute_aupr(image_level$filter_150_skeletonized, image_level$label, 'PR: Segmentations: zero all pixels < 150 and skeletonize')
compute_auroc(image_level$filter_150_skeletonized, image_level$label, 'ROC: Segmentations: zero all pixels < 150 and skeletonize')
confusion_matrix(image_level, 'filter_150_skeletonized')Confusion Matrix and Statistics
Reference
Prediction 0 1
0 641 162
1 186 361
Accuracy : 0.7422
95% CI : (0.718, 0.7654)
No Information Rate : 0.6126
P-Value [Acc > NIR] : <2e-16
Kappa : 0.4614
Mcnemar's Test P-Value : 0.2176
Sensitivity : 0.6902
Specificity : 0.7751
Pos Pred Value : 0.6600
Neg Pred Value : 0.7983
Prevalence : 0.3874
Detection Rate : 0.2674
Detection Prevalence : 0.4052
Balanced Accuracy : 0.7327
'Positive' Class : 1
Segmentations: zero all pixels < 200
compute_aupr(image_level$filter_200, image_level$label, 'PR: Segmentations: zero all pixels < 200')
compute_auroc(image_level$filter_200, image_level$label, 'ROC: Segmentations: zero all pixels < 200')
confusion_matrix(image_level, 'filter_200')Confusion Matrix and Statistics
Reference
Prediction 0 1
0 548 121
1 279 402
Accuracy : 0.7037
95% CI : (0.6786, 0.728)
No Information Rate : 0.6126
P-Value [Acc > NIR] : 1.64e-12
Kappa : 0.4086
Mcnemar's Test P-Value : 4.16e-15
Sensitivity : 0.7686
Specificity : 0.6626
Pos Pred Value : 0.5903
Neg Pred Value : 0.8191
Prevalence : 0.3874
Detection Rate : 0.2978
Detection Prevalence : 0.5044
Balanced Accuracy : 0.7156
'Positive' Class : 1
compute_aupr(image_level$filter_200_binarized, image_level$label, 'PR: Segmentations: zero all pixels < 200 and binarize')
compute_auroc(image_level$filter_200_binarized, image_level$label, 'ROC: Segmentations: zero all pixels < 200 and binarize')
confusion_matrix(image_level, 'filter_200_binarized')Confusion Matrix and Statistics
Reference
Prediction 0 1
0 661 233
1 166 290
Accuracy : 0.7044
95% CI : (0.6793, 0.7287)
No Information Rate : 0.6126
P-Value [Acc > NIR] : 1.083e-12
Kappa : 0.3623
Mcnemar's Test P-Value : 0.0009527
Sensitivity : 0.5545
Specificity : 0.7993
Pos Pred Value : 0.6360
Neg Pred Value : 0.7394
Prevalence : 0.3874
Detection Rate : 0.2148
Detection Prevalence : 0.3378
Balanced Accuracy : 0.6769
'Positive' Class : 1
compute_aupr(image_level$filter_200_skeletonized, image_level$label, 'PR: Segmentations: zero all pixels < 200 and skeletonize')
compute_auroc(image_level$filter_200_skeletonized, image_level$label, 'ROC: Segmentations: zero all pixels < 200 and skeletonize')
confusion_matrix(image_level, 'filter_200_skeletonized')Confusion Matrix and Statistics
Reference
Prediction 0 1
0 766 365
1 61 158
Accuracy : 0.6844
95% CI : (0.6589, 0.7092)
No Information Rate : 0.6126
P-Value [Acc > NIR] : 2.354e-08
Kappa : 0.2557
Mcnemar's Test P-Value : < 2.2e-16
Sensitivity : 0.3021
Specificity : 0.9262
Pos Pred Value : 0.7215
Neg Pred Value : 0.6773
Prevalence : 0.3874
Detection Rate : 0.1170
Detection Prevalence : 0.1622
Balanced Accuracy : 0.6142
'Positive' Class : 1
Segmentations: zero all pixels < 210
compute_aupr(image_level$filter_210, image_level$label, 'PR: Segmentations: zero all pixels < 210')
compute_auroc(image_level$filter_210, image_level$label, 'ROC: Segmentations: zero all pixels < 210')
confusion_matrix(image_level, 'filter_210')Confusion Matrix and Statistics
Reference
Prediction 0 1
0 712 326
1 115 197
Accuracy : 0.6733
95% CI : (0.6476, 0.6983)
No Information Rate : 0.6126
P-Value [Acc > NIR] : 2.096e-06
Kappa : 0.2566
Mcnemar's Test P-Value : < 2.2e-16
Sensitivity : 0.3767
Specificity : 0.8609
Pos Pred Value : 0.6314
Neg Pred Value : 0.6859
Prevalence : 0.3874
Detection Rate : 0.1459
Detection Prevalence : 0.2311
Balanced Accuracy : 0.6188
'Positive' Class : 1
compute_aupr(image_level$filter_210_binarized, image_level$label, 'PR: Segmentations: zero all pixels < 210 and binarize')
compute_auroc(image_level$filter_210_binarized, image_level$label, 'ROC: Segmentations: zero all pixels < 210 and binarize')
confusion_matrix(image_level, 'filter_210_binarized')Confusion Matrix and Statistics
Reference
Prediction 0 1
0 639 203
1 188 320
Accuracy : 0.7104
95% CI : (0.6854, 0.7345)
No Information Rate : 0.6126
P-Value [Acc > NIR] : 3.463e-14
Kappa : 0.3866
Mcnemar's Test P-Value : 0.4789
Sensitivity : 0.6119
Specificity : 0.7727
Pos Pred Value : 0.6299
Neg Pred Value : 0.7589
Prevalence : 0.3874
Detection Rate : 0.2370
Detection Prevalence : 0.3763
Balanced Accuracy : 0.6923
'Positive' Class : 1
compute_aupr(image_level$filter_210_skeletonized, image_level$label, 'PR: Segmentations: zero all pixels < 210 and skeletonize')
compute_auroc(image_level$filter_210_skeletonized, image_level$label, 'ROC: Segmentations: zero all pixels < 210 and skeletonize')
confusion_matrix(image_level, 'filter_210_skeletonized')Confusion Matrix and Statistics
Reference
Prediction 0 1
0 562 160
1 265 363
Accuracy : 0.6852
95% CI : (0.6597, 0.7099)
No Information Rate : 0.6126
P-Value [Acc > NIR] : 1.699e-08
Kappa : 0.3603
Mcnemar's Test P-Value : 4.541e-07
Sensitivity : 0.6941
Specificity : 0.6796
Pos Pred Value : 0.5780
Neg Pred Value : 0.7784
Prevalence : 0.3874
Detection Rate : 0.2689
Detection Prevalence : 0.4652
Balanced Accuracy : 0.6868
'Positive' Class : 1
Segmentations: zero all pixels < 220
compute_aupr(image_level$filter_220, image_level$label, 'PR: Segmentations: zero all pixels < 220')
compute_auroc(image_level$filter_220, image_level$label, 'ROC: Segmentations: zero all pixels < 220')
confusion_matrix(image_level, 'filter_220')Confusion Matrix and Statistics
Reference
Prediction 0 1
0 576 224
1 251 299
Accuracy : 0.6481
95% CI : (0.622, 0.6736)
No Information Rate : 0.6126
P-Value [Acc > NIR] : 0.003819
Kappa : 0.2657
Mcnemar's Test P-Value : 0.232884
Sensitivity : 0.5717
Specificity : 0.6965
Pos Pred Value : 0.5436
Neg Pred Value : 0.7200
Prevalence : 0.3874
Detection Rate : 0.2215
Detection Prevalence : 0.4074
Balanced Accuracy : 0.6341
'Positive' Class : 1
compute_aupr(image_level$filter_220_binarized, image_level$label, 'PR: Segmentations: zero all pixels < 220 and binarize')
compute_auroc(image_level$filter_220_binarized, image_level$label, 'ROC: Segmentations: zero all pixels < 220 and binarize')
confusion_matrix(image_level, 'filter_220_binarized')Confusion Matrix and Statistics
Reference
Prediction 0 1
0 545 162
1 282 361
Accuracy : 0.6711
95% CI : (0.6453, 0.6961)
No Information Rate : 0.6126
P-Value [Acc > NIR] : 4.710e-06
Kappa : 0.3351
Mcnemar's Test P-Value : 1.628e-08
Sensitivity : 0.6902
Specificity : 0.6590
Pos Pred Value : 0.5614
Neg Pred Value : 0.7709
Prevalence : 0.3874
Detection Rate : 0.2674
Detection Prevalence : 0.4763
Balanced Accuracy : 0.6746
'Positive' Class : 1
compute_aupr(image_level$filter_220_skeletonized, image_level$label, 'PR: Segmentations: zero all pixels < 220 and skeletonize')
compute_auroc(image_level$filter_220_skeletonized, image_level$label, 'ROC: Segmentations: zero all pixels < 220 and skeletonize')
confusion_matrix(image_level, 'filter_220_skeletonized')Confusion Matrix and Statistics
Reference
Prediction 0 1
0 800 450
1 27 73
Accuracy : 0.6467
95% CI : (0.6205, 0.6722)
No Information Rate : 0.6126
P-Value [Acc > NIR] : 0.005317
Kappa : 0.1256
Mcnemar's Test P-Value : < 2.2e-16
Sensitivity : 0.13958
Specificity : 0.96735
Pos Pred Value : 0.73000
Neg Pred Value : 0.64000
Prevalence : 0.38741
Detection Rate : 0.05407
Detection Prevalence : 0.07407
Balanced Accuracy : 0.55347
'Positive' Class : 1
Segmentations: zero all pixels < 230
compute_aupr(image_level$filter_230, image_level$label, 'PR: Segmentations: zero all pixels < 230')
compute_auroc(image_level$filter_230, image_level$label, 'ROC: Segmentations: zero all pixels < 230')
confusion_matrix(image_level, 'filter_230')Confusion Matrix and Statistics
Reference
Prediction 0 1
0 464 160
1 363 363
Accuracy : 0.6126
95% CI : (0.586, 0.6387)
No Information Rate : 0.6126
P-Value [Acc > NIR] : 0.512
Kappa : 0.2381
Mcnemar's Test P-Value : <2e-16
Sensitivity : 0.6941
Specificity : 0.5611
Pos Pred Value : 0.5000
Neg Pred Value : 0.7436
Prevalence : 0.3874
Detection Rate : 0.2689
Detection Prevalence : 0.5378
Balanced Accuracy : 0.6276
'Positive' Class : 1
compute_aupr(image_level$filter_230_binarized, image_level$label, 'PR: Segmentations: zero all pixels < 230 and binarize')
compute_auroc(image_level$filter_230_binarized, image_level$label, 'ROC: Segmentations: zero all pixels < 230 and binarize')
confusion_matrix(image_level, 'filter_230_binarized')Confusion Matrix and Statistics
Reference
Prediction 0 1
0 246 46
1 581 477
Accuracy : 0.5356
95% CI : (0.5085, 0.5624)
No Information Rate : 0.6126
P-Value [Acc > NIR] : 1
Kappa : 0.1764
Mcnemar's Test P-Value : <2e-16
Sensitivity : 0.9120
Specificity : 0.2975
Pos Pred Value : 0.4509
Neg Pred Value : 0.8425
Prevalence : 0.3874
Detection Rate : 0.3533
Detection Prevalence : 0.7837
Balanced Accuracy : 0.6048
'Positive' Class : 1
compute_aupr(image_level$filter_230_skeletonized, image_level$label, 'PR: Segmentations: zero all pixels < 230 and skeletonize')
compute_auroc(image_level$filter_230_skeletonized, image_level$label, 'ROC: Segmentations: zero all pixels < 230 and skeletonize')
confusion_matrix(image_level, 'filter_230_skeletonized')Confusion Matrix and Statistics
Reference
Prediction 0 1
0 365 110
1 462 413
Accuracy : 0.5763
95% CI : (0.5494, 0.6028)
No Information Rate : 0.6126
P-Value [Acc > NIR] : 0.997
Kappa : 0.2056
Mcnemar's Test P-Value : <2e-16
Sensitivity : 0.7897
Specificity : 0.4414
Pos Pred Value : 0.4720
Neg Pred Value : 0.7684
Prevalence : 0.3874
Detection Rate : 0.3059
Detection Prevalence : 0.6481
Balanced Accuracy : 0.6155
'Positive' Class : 1
Segmentations: zero all pixels < 240
compute_aupr(image_level$filter_240, image_level$label, 'PR: Segmentations: zero all pixels < 240')
compute_auroc(image_level$filter_240, image_level$label, 'ROC: Segmentations: zero all pixels < 240')
confusion_matrix(image_level, 'filter_240')Confusion Matrix and Statistics
Reference
Prediction 0 1
0 414 183
1 413 340
Accuracy : 0.5585
95% CI : (0.5316, 0.5852)
No Information Rate : 0.6126
P-Value [Acc > NIR] : 1
Kappa : 0.1394
Mcnemar's Test P-Value : <2e-16
Sensitivity : 0.6501
Specificity : 0.5006
Pos Pred Value : 0.4515
Neg Pred Value : 0.6935
Prevalence : 0.3874
Detection Rate : 0.2519
Detection Prevalence : 0.5578
Balanced Accuracy : 0.5754
'Positive' Class : 1
compute_aupr(image_level$filter_240_binarized, image_level$label, 'PR: Segmentations: zero all pixels < 240 and binarize')
compute_auroc(image_level$filter_240_binarized, image_level$label, 'ROC: Segmentations: zero all pixels < 240 and binarize')
confusion_matrix(image_level, 'filter_240_binarized')Confusion Matrix and Statistics
Reference
Prediction 0 1
0 258 83
1 569 440
Accuracy : 0.517
95% CI : (0.49, 0.544)
No Information Rate : 0.6126
P-Value [Acc > NIR] : 1
Kappa : 0.1309
Mcnemar's Test P-Value : <2e-16
Sensitivity : 0.8413
Specificity : 0.3120
Pos Pred Value : 0.4361
Neg Pred Value : 0.7566
Prevalence : 0.3874
Detection Rate : 0.3259
Detection Prevalence : 0.7474
Balanced Accuracy : 0.5766
'Positive' Class : 1
compute_aupr(image_level$filter_240_skeletonized, image_level$label, 'PR: Segmentations: zero all pixels < 240 and skeletonize')
compute_auroc(image_level$filter_240_skeletonized, image_level$label, 'ROC: Segmentations: zero all pixels < 240 and skeletonize')
confusion_matrix(image_level, 'filter_240_skeletonized')Confusion Matrix and Statistics
Reference
Prediction 0 1
0 279 103
1 548 420
Accuracy : 0.5178
95% CI : (0.4907, 0.5447)
No Information Rate : 0.6126
P-Value [Acc > NIR] : 1
Kappa : 0.1214
Mcnemar's Test P-Value : <2e-16
Sensitivity : 0.8031
Specificity : 0.3374
Pos Pred Value : 0.4339
Neg Pred Value : 0.7304
Prevalence : 0.3874
Detection Rate : 0.3111
Detection Prevalence : 0.7170
Balanced Accuracy : 0.5702
'Positive' Class : 1
Segmentations: zero all pixels < 250
compute_aupr(image_level$filter_250, image_level$label, 'PR: Segmentations: zero all pixels < 250')
compute_auroc(image_level$filter_250, image_level$label, 'ROC: Segmentations: zero all pixels < 250')
confusion_matrix(image_level, 'filter_250')Confusion Matrix and Statistics
Reference
Prediction 0 1
0 51 9
1 776 514
Accuracy : 0.4185
95% CI : (0.392, 0.4454)
No Information Rate : 0.6126
P-Value [Acc > NIR] : 1
Kappa : 0.035
Mcnemar's Test P-Value : <2e-16
Sensitivity : 0.98279
Specificity : 0.06167
Pos Pred Value : 0.39845
Neg Pred Value : 0.85000
Prevalence : 0.38741
Detection Rate : 0.38074
Detection Prevalence : 0.95556
Balanced Accuracy : 0.52223
'Positive' Class : 1
compute_aupr(image_level$filter_250_binarized, image_level$label, 'PR: Segmentations: zero all pixels < 250 and binarize')
compute_auroc(image_level$filter_250_binarized, image_level$label, 'ROC: Segmentations: zero all pixels < 250 and binarize')
confusion_matrix(image_level, 'filter_250_binarized')Confusion Matrix and Statistics
Reference
Prediction 0 1
0 50 8
1 777 515
Accuracy : 0.4185
95% CI : (0.392, 0.4454)
No Information Rate : 0.6126
P-Value [Acc > NIR] : 1
Kappa : 0.0356
Mcnemar's Test P-Value : <2e-16
Sensitivity : 0.98470
Specificity : 0.06046
Pos Pred Value : 0.39861
Neg Pred Value : 0.86207
Prevalence : 0.38741
Detection Rate : 0.38148
Detection Prevalence : 0.95704
Balanced Accuracy : 0.52258
'Positive' Class : 1
compute_aupr(image_level$filter_250_skeletonized, image_level$label, 'PR: Segmentations: zero all pixels < 250 and skeletonize')
compute_auroc(image_level$filter_250_skeletonized, image_level$label, 'ROC: Segmentations: zero all pixels < 250 and skeletonize')
confusion_matrix(image_level, 'filter_250_skeletonized')Confusion Matrix and Statistics
Reference
Prediction 0 1
0 53 9
1 774 514
Accuracy : 0.42
95% CI : (0.3935, 0.4468)
No Information Rate : 0.6126
P-Value [Acc > NIR] : 1
Kappa : 0.0369
Mcnemar's Test P-Value : <2e-16
Sensitivity : 0.98279
Specificity : 0.06409
Pos Pred Value : 0.39907
Neg Pred Value : 0.85484
Prevalence : 0.38741
Detection Rate : 0.38074
Detection Prevalence : 0.95407
Balanced Accuracy : 0.52344
'Positive' Class : 1
Segmentations: zero all pixels < 257
compute_aupr(image_level$filter_257, image_level$label, 'PR: Segmentations: zero all pixels < 257')
compute_auroc(image_level$filter_257, image_level$label, 'ROC: Segmentations: zero all pixels < 257')
confusion_matrix(image_level, 'filter_257')Warning in confusionMatrix.default(preds, df$label, positive = "1") :
Levels are not in the same order for reference and data. Refactoring data to match.Confusion Matrix and Statistics
Reference
Prediction 0 1
0 827 523
1 0 0
Accuracy : 0.6126
95% CI : (0.586, 0.6387)
No Information Rate : 0.6126
P-Value [Acc > NIR] : 0.512
Kappa : 0
Mcnemar's Test P-Value : <2e-16
Sensitivity : 0.0000
Specificity : 1.0000
Pos Pred Value : NaN
Neg Pred Value : 0.6126
Prevalence : 0.3874
Detection Rate : 0.0000
Detection Prevalence : 0.0000
Balanced Accuracy : 0.5000
'Positive' Class : 1
compute_aupr(image_level$filter_257_binarized, image_level$label, 'PR: Segmentations: zero all pixels < 257 and binarize')
compute_auroc(image_level$filter_257_binarized, image_level$label, 'ROC: Segmentations: zero all pixels < 257 and binarize')
confusion_matrix(image_level, 'filter_257_binarized')Warning in confusionMatrix.default(preds, df$label, positive = "1") :
Levels are not in the same order for reference and data. Refactoring data to match.Confusion Matrix and Statistics
Reference
Prediction 0 1
0 827 523
1 0 0
Accuracy : 0.6126
95% CI : (0.586, 0.6387)
No Information Rate : 0.6126
P-Value [Acc > NIR] : 0.512
Kappa : 0
Mcnemar's Test P-Value : <2e-16
Sensitivity : 0.0000
Specificity : 1.0000
Pos Pred Value : NaN
Neg Pred Value : 0.6126
Prevalence : 0.3874
Detection Rate : 0.0000
Detection Prevalence : 0.0000
Balanced Accuracy : 0.5000
'Positive' Class : 1
compute_aupr(image_level$filter_257_skeletonized, image_level$label, 'PR: Segmentations: zero all pixels < 257 and skeletonize')
compute_auroc(image_level$filter_257_skeletonized, image_level$label, 'ROC: Segmentations: zero all pixels < 257 and skeletonize')
confusion_matrix(image_level, 'filter_257_skeletonized')Warning in confusionMatrix.default(preds, df$label, positive = "1") :
Levels are not in the same order for reference and data. Refactoring data to match.Confusion Matrix and Statistics
Reference
Prediction 0 1
0 827 523
1 0 0
Accuracy : 0.6126
95% CI : (0.586, 0.6387)
No Information Rate : 0.6126
P-Value [Acc > NIR] : 0.512
Kappa : 0
Mcnemar's Test P-Value : <2e-16
Sensitivity : 0.0000
Specificity : 1.0000
Pos Pred Value : NaN
Neg Pred Value : 0.6126
Prevalence : 0.3874
Detection Rate : 0.0000
Detection Prevalence : 0.0000
Balanced Accuracy : 0.5000
'Positive' Class : 1
Segmentations: zero all pixels > 10
compute_aupr(image_level$filter_10, image_level$label, 'PR: Segmentations: zero all pixels < 10')
compute_auroc(image_level$filter_10, image_level$label, 'ROC: Segmentations: zero all pixels < 10')
confusion_matrix(image_level, 'filter_10')Confusion Matrix and Statistics
Reference
Prediction 0 1
0 763 196
1 64 327
Accuracy : 0.8074
95% CI : (0.7853, 0.8281)
No Information Rate : 0.6126
P-Value [Acc > NIR] : < 2e-16
Kappa : 0.5745
Mcnemar's Test P-Value : 4.5e-16
Sensitivity : 0.6252
Specificity : 0.9226
Pos Pred Value : 0.8363
Neg Pred Value : 0.7956
Prevalence : 0.3874
Detection Rate : 0.2422
Detection Prevalence : 0.2896
Balanced Accuracy : 0.7739
'Positive' Class : 1
compute_aupr(image_level$filter_10_binarized, image_level$label, 'PR: Segmentations: zero all pixels < 10 and binarize')
compute_auroc(image_level$filter_10_binarized, image_level$label, 'ROC: Segmentations: zero all pixels < 10 and binarize')
confusion_matrix(image_level, 'filter_10_binarized')Confusion Matrix and Statistics
Reference
Prediction 0 1
0 621 7
1 206 516
Accuracy : 0.8422
95% CI : (0.8217, 0.8613)
No Information Rate : 0.6126
P-Value [Acc > NIR] : < 2.2e-16
Kappa : 0.6893
Mcnemar's Test P-Value : < 2.2e-16
Sensitivity : 0.9866
Specificity : 0.7509
Pos Pred Value : 0.7147
Neg Pred Value : 0.9889
Prevalence : 0.3874
Detection Rate : 0.3822
Detection Prevalence : 0.5348
Balanced Accuracy : 0.8688
'Positive' Class : 1
compute_aupr(image_level$filter_10_skeletonized, image_level$label, 'PR: Segmentations: zero all pixels < 10 and skeletonize')
compute_auroc(image_level$filter_10_skeletonized, image_level$label, 'ROC: Segmentations: zero all pixels < 10 and skeletonize')
confusion_matrix(image_level, 'filter_10_skeletonized')Confusion Matrix and Statistics
Reference
Prediction 0 1
0 696 43
1 131 480
Accuracy : 0.8711
95% CI : (0.8521, 0.8885)
No Information Rate : 0.6126
P-Value [Acc > NIR] : < 2.2e-16
Kappa : 0.7366
Mcnemar's Test P-Value : 4.24e-11
Sensitivity : 0.9178
Specificity : 0.8416
Pos Pred Value : 0.7856
Neg Pred Value : 0.9418
Prevalence : 0.3874
Detection Rate : 0.3556
Detection Prevalence : 0.4526
Balanced Accuracy : 0.8797
'Positive' Class : 1
Segmentations: zero all pixels < 75 and > 150
compute_aupr(image_level$filter_75, image_level$label, 'PR: Segmentations: zero all pixels < 75 and > 150')
compute_auroc(image_level$filter_75, image_level$label, 'ROC: Segmentations: zero all pixels < 75 and > 150')
confusion_matrix(image_level, 'filter_75')Confusion Matrix and Statistics
Reference
Prediction 0 1
0 597 77
1 230 446
Accuracy : 0.7726
95% CI : (0.7493, 0.7947)
No Information Rate : 0.6126
P-Value [Acc > NIR] : < 2.2e-16
Kappa : 0.5453
Mcnemar's Test P-Value : < 2.2e-16
Sensitivity : 0.8528
Specificity : 0.7219
Pos Pred Value : 0.6598
Neg Pred Value : 0.8858
Prevalence : 0.3874
Detection Rate : 0.3304
Detection Prevalence : 0.5007
Balanced Accuracy : 0.7873
'Positive' Class : 1
compute_aupr(image_level$filter_75_binarized, image_level$label, 'PR: Segmentations: zero all pixels < 75 and > 150 and binarize')
compute_auroc(image_level$filter_75_binarized, image_level$label, 'ROC: Segmentations: zero all pixels < 75 and > 150 and binarize')
confusion_matrix(image_level, 'filter_75_binarized')Confusion Matrix and Statistics
Reference
Prediction 0 1
0 545 69
1 282 454
Accuracy : 0.74
95% CI : (0.7157, 0.7632)
No Information Rate : 0.6126
P-Value [Acc > NIR] : < 2.2e-16
Kappa : 0.4904
Mcnemar's Test P-Value : < 2.2e-16
Sensitivity : 0.8681
Specificity : 0.6590
Pos Pred Value : 0.6168
Neg Pred Value : 0.8876
Prevalence : 0.3874
Detection Rate : 0.3363
Detection Prevalence : 0.5452
Balanced Accuracy : 0.7635
'Positive' Class : 1
compute_aupr(image_level$filter_75_skeletonized, image_level$label, 'PR: Segmentations: zero all pixels < 75 and > 150 and skeletonize')
compute_auroc(image_level$filter_75_skeletonized, image_level$label, 'ROC: Segmentations: zero all pixels < 75 and > 150 and skeletonize')
confusion_matrix(image_level, 'filter_75_skeletonized')Confusion Matrix and Statistics
Reference
Prediction 0 1
0 726 224
1 101 299
Accuracy : 0.7593
95% CI : (0.7355, 0.7819)
No Information Rate : 0.6126
P-Value [Acc > NIR] : < 2.2e-16
Kappa : 0.4699
Mcnemar's Test P-Value : 1.312e-11
Sensitivity : 0.5717
Specificity : 0.8779
Pos Pred Value : 0.7475
Neg Pred Value : 0.7642
Prevalence : 0.3874
Detection Rate : 0.2215
Detection Prevalence : 0.2963
Balanced Accuracy : 0.7248
'Positive' Class : 1
Subject-level Analysis
RetCam Images
compute_aupr(subject_level$retcam, subject_level$label, 'PR: Raw RetCam Images')
compute_auroc(subject_level$retcam, subject_level$label, 'ROC: Raw RetCam Images')
confusion_matrix(subject_level, 'retcam')Confusion Matrix and Statistics
Reference
Prediction 0 1
0 43 0
1 0 27
Accuracy : 1
95% CI : (0.9487, 1)
No Information Rate : 0.6143
P-Value [Acc > NIR] : 1.534e-15
Kappa : 1
Mcnemar's Test P-Value : NA
Sensitivity : 1.0000
Specificity : 1.0000
Pos Pred Value : 1.0000
Neg Pred Value : 1.0000
Prevalence : 0.3857
Detection Rate : 0.3857
Detection Prevalence : 0.3857
Balanced Accuracy : 1.0000
'Positive' Class : 1
compute_aupr(subject_level$retcam_random, subject_level$label, 'PR: Raw RetCam Images - Shuffled Labels')
compute_auroc(subject_level$retcam_random, subject_level$label, 'ROC: Raw RetCam Images - Shuffled Labels')
confusion_matrix(subject_level, 'retcam_random')Confusion Matrix and Statistics
Reference
Prediction 0 1
0 24 2
1 19 25
Accuracy : 0.7
95% CI : (0.5787, 0.8038)
No Information Rate : 0.6143
P-Value [Acc > NIR] : 0.0869051
Kappa : 0.4333
Mcnemar's Test P-Value : 0.0004803
Sensitivity : 0.9259
Specificity : 0.5581
Pos Pred Value : 0.5682
Neg Pred Value : 0.9231
Prevalence : 0.3857
Detection Rate : 0.3571
Detection Prevalence : 0.6286
Balanced Accuracy : 0.7420
'Positive' Class : 1
Segmentations: zero all pixels < 0
compute_aupr(subject_level$filter_0, subject_level$label, 'PR: Segmentations: zero all pixels < 0')
compute_auroc(subject_level$filter_0, subject_level$label, 'ROC: Segmentations: zero all pixels < 0')
confusion_matrix(subject_level, 'filter_0')Confusion Matrix and Statistics
Reference
Prediction 0 1
0 42 2
1 1 25
Accuracy : 0.9571
95% CI : (0.8798, 0.9911)
No Information Rate : 0.6143
P-Value [Acc > NIR] : 2.232e-11
Kappa : 0.9089
Mcnemar's Test P-Value : 1
Sensitivity : 0.9259
Specificity : 0.9767
Pos Pred Value : 0.9615
Neg Pred Value : 0.9545
Prevalence : 0.3857
Detection Rate : 0.3571
Detection Prevalence : 0.3714
Balanced Accuracy : 0.9513
'Positive' Class : 1
compute_aupr(subject_level$filter_0_binarized, subject_level$label, 'PR: Segmentations: zero all pixels < 0 and binarize')
compute_auroc(subject_level$filter_0_binarized, subject_level$label, 'ROC: Segmentations: zero all pixels < 0 and binarize')
confusion_matrix(subject_level, 'filter_0_binarized')Confusion Matrix and Statistics
Reference
Prediction 0 1
0 41 0
1 2 27
Accuracy : 0.9714
95% CI : (0.9006, 0.9965)
No Information Rate : 0.6143
P-Value [Acc > NIR] : 1.53e-12
Kappa : 0.9405
Mcnemar's Test P-Value : 0.4795
Sensitivity : 1.0000
Specificity : 0.9535
Pos Pred Value : 0.9310
Neg Pred Value : 1.0000
Prevalence : 0.3857
Detection Rate : 0.3857
Detection Prevalence : 0.4143
Balanced Accuracy : 0.9767
'Positive' Class : 1
compute_aupr(subject_level$filter_0_skeletonized, subject_level$label, 'PR: Segmentations: zero all pixels < 0 and skeletonize')
compute_auroc(subject_level$filter_0_skeletonized, subject_level$label, 'ROC: Segmentations: zero all pixels < 0 and skeletonize')
confusion_matrix(subject_level, 'filter_0_skeletonized')Confusion Matrix and Statistics
Reference
Prediction 0 1
0 40 0
1 3 27
Accuracy : 0.9571
95% CI : (0.8798, 0.9911)
No Information Rate : 0.6143
P-Value [Acc > NIR] : 2.232e-11
Kappa : 0.9114
Mcnemar's Test P-Value : 0.2482
Sensitivity : 1.0000
Specificity : 0.9302
Pos Pred Value : 0.9000
Neg Pred Value : 1.0000
Prevalence : 0.3857
Detection Rate : 0.3857
Detection Prevalence : 0.4286
Balanced Accuracy : 0.9651
'Positive' Class : 1
compute_aupr(subject_level$filter_0_random, subject_level$label, 'PR: Segmentations: zero all pixels < 0 - Shuffled Labels')
compute_auroc(subject_level$filter_0_random, subject_level$label, 'ROC: Segmentations: zero all pixels < 0 - Shuffled Labels')
confusion_matrix(subject_level, 'filter_0_random')Warning in confusionMatrix.default(preds, df$label, positive = "1") :
Levels are not in the same order for reference and data. Refactoring data to match.Confusion Matrix and Statistics
Reference
Prediction 0 1
0 43 27
1 0 0
Accuracy : 0.6143
95% CI : (0.4903, 0.7283)
No Information Rate : 0.6143
P-Value [Acc > NIR] : 0.5525
Kappa : 0
Mcnemar's Test P-Value : 5.624e-07
Sensitivity : 0.0000
Specificity : 1.0000
Pos Pred Value : NaN
Neg Pred Value : 0.6143
Prevalence : 0.3857
Detection Rate : 0.0000
Detection Prevalence : 0.0000
Balanced Accuracy : 0.5000
'Positive' Class : 1
Segmentations: zero all pixels < 50
compute_aupr(subject_level$filter_50, subject_level$label, 'PR: Segmentations: zero all pixels < 50')
compute_auroc(subject_level$filter_50, subject_level$label, 'ROC: Segmentations: zero all pixels < 50')
confusion_matrix(subject_level, 'filter_50')Confusion Matrix and Statistics
Reference
Prediction 0 1
0 36 0
1 7 27
Accuracy : 0.9
95% CI : (0.8048, 0.9588)
No Information Rate : 0.6143
P-Value [Acc > NIR] : 8.517e-08
Kappa : 0.7987
Mcnemar's Test P-Value : 0.02334
Sensitivity : 1.0000
Specificity : 0.8372
Pos Pred Value : 0.7941
Neg Pred Value : 1.0000
Prevalence : 0.3857
Detection Rate : 0.3857
Detection Prevalence : 0.4857
Balanced Accuracy : 0.9186
'Positive' Class : 1
compute_aupr(subject_level$filter_50_binarized, subject_level$label, 'PR: Segmentations: zero all pixels < 50 and binarize')
compute_auroc(subject_level$filter_50_binarized, subject_level$label, 'ROC: Segmentations: zero all pixels < 50 and binarize')
confusion_matrix(subject_level, 'filter_0_binarized')Confusion Matrix and Statistics
Reference
Prediction 0 1
0 41 0
1 2 27
Accuracy : 0.9714
95% CI : (0.9006, 0.9965)
No Information Rate : 0.6143
P-Value [Acc > NIR] : 1.53e-12
Kappa : 0.9405
Mcnemar's Test P-Value : 0.4795
Sensitivity : 1.0000
Specificity : 0.9535
Pos Pred Value : 0.9310
Neg Pred Value : 1.0000
Prevalence : 0.3857
Detection Rate : 0.3857
Detection Prevalence : 0.4143
Balanced Accuracy : 0.9767
'Positive' Class : 1
compute_aupr(subject_level$filter_50_skeletonized, subject_level$label, 'PR: Segmentations: zero all pixels < 50 and skeletonize')
compute_auroc(subject_level$filter_50_skeletonized, subject_level$label, 'ROC: Segmentations: zero all pixels < 50 and skeletonize')
confusion_matrix(subject_level, 'filter_50_skeletonized')Confusion Matrix and Statistics
Reference
Prediction 0 1
0 19 0
1 24 27
Accuracy : 0.6571
95% CI : (0.534, 0.7665)
No Information Rate : 0.6143
P-Value [Acc > NIR] : 0.2718
Kappa : 0.3792
Mcnemar's Test P-Value : 2.668e-06
Sensitivity : 1.0000
Specificity : 0.4419
Pos Pred Value : 0.5294
Neg Pred Value : 1.0000
Prevalence : 0.3857
Detection Rate : 0.3857
Detection Prevalence : 0.7286
Balanced Accuracy : 0.7209
'Positive' Class : 1
Segmentations: zero all pixels < 100
compute_aupr(subject_level$filter_100, subject_level$label, 'PR: Segmentations: zero all pixels < 100')
compute_auroc(subject_level$filter_100, subject_level$label, 'ROC: Segmentations: zero all pixels < 100')
confusion_matrix(subject_level, 'filter_100')Confusion Matrix and Statistics
Reference
Prediction 0 1
0 43 10
1 0 17
Accuracy : 0.8571
95% CI : (0.7529, 0.9293)
No Information Rate : 0.6143
P-Value [Acc > NIR] : 7.748e-06
Kappa : 0.6762
Mcnemar's Test P-Value : 0.004427
Sensitivity : 0.6296
Specificity : 1.0000
Pos Pred Value : 1.0000
Neg Pred Value : 0.8113
Prevalence : 0.3857
Detection Rate : 0.2429
Detection Prevalence : 0.2429
Balanced Accuracy : 0.8148
'Positive' Class : 1
compute_aupr(subject_level$filter_100_binarized, subject_level$label, 'PR: Segmentations: zero all pixels < 100 and binarize')
compute_auroc(subject_level$filter_100_binarized, subject_level$label, 'ROC: Segmentations: zero all pixels < 100 and binarize')
confusion_matrix(subject_level, 'filter_100_binarized')Confusion Matrix and Statistics
Reference
Prediction 0 1
0 39 4
1 4 23
Accuracy : 0.8857
95% CI : (0.7872, 0.9493)
No Information Rate : 0.6143
P-Value [Acc > NIR] : 4.352e-07
Kappa : 0.7588
Mcnemar's Test P-Value : 1
Sensitivity : 0.8519
Specificity : 0.9070
Pos Pred Value : 0.8519
Neg Pred Value : 0.9070
Prevalence : 0.3857
Detection Rate : 0.3286
Detection Prevalence : 0.3857
Balanced Accuracy : 0.8794
'Positive' Class : 1
compute_aupr(subject_level$filter_100_skeletonized, subject_level$label, 'PR: Segmentations: zero all pixels < 100 and skeletonize')
compute_auroc(subject_level$filter_100_skeletonized, subject_level$label, 'ROC: Segmentations: zero all pixels < 100 and skeletonize')
confusion_matrix(subject_level, 'filter_100_skeletonized')Confusion Matrix and Statistics
Reference
Prediction 0 1
0 38 3
1 5 24
Accuracy : 0.8857
95% CI : (0.7872, 0.9493)
No Information Rate : 0.6143
P-Value [Acc > NIR] : 4.352e-07
Kappa : 0.7621
Mcnemar's Test P-Value : 0.7237
Sensitivity : 0.8889
Specificity : 0.8837
Pos Pred Value : 0.8276
Neg Pred Value : 0.9268
Prevalence : 0.3857
Detection Rate : 0.3429
Detection Prevalence : 0.4143
Balanced Accuracy : 0.8863
'Positive' Class : 1
Segmentations: zero all pixels < 150
compute_aupr(subject_level$filter_150, subject_level$label, 'PR: Segmentations: zero all pixels < 150')
compute_auroc(subject_level$filter_150, subject_level$label, 'ROC: Segmentations: zero all pixels < 150')
confusion_matrix(subject_level, 'filter_150')Confusion Matrix and Statistics
Reference
Prediction 0 1
0 37 0
1 6 27
Accuracy : 0.9143
95% CI : (0.8227, 0.9679)
No Information Rate : 0.6143
P-Value [Acc > NIR] : 1.438e-08
Kappa : 0.8263
Mcnemar's Test P-Value : 0.04123
Sensitivity : 1.0000
Specificity : 0.8605
Pos Pred Value : 0.8182
Neg Pred Value : 1.0000
Prevalence : 0.3857
Detection Rate : 0.3857
Detection Prevalence : 0.4714
Balanced Accuracy : 0.9302
'Positive' Class : 1
compute_aupr(subject_level$filter_150_binarized, subject_level$label, 'PR: Segmentations: zero all pixels < 150 and binarize')
compute_auroc(subject_level$filter_150_binarized, subject_level$label, 'ROC: Segmentations: zero all pixels < 150 and binarize')
confusion_matrix(subject_level, 'filter_150_binarized')Confusion Matrix and Statistics
Reference
Prediction 0 1
0 43 5
1 0 22
Accuracy : 0.9286
95% CI : (0.8411, 0.9764)
No Information Rate : 0.6143
P-Value [Acc > NIR] : 2.054e-09
Kappa : 0.8439
Mcnemar's Test P-Value : 0.07364
Sensitivity : 0.8148
Specificity : 1.0000
Pos Pred Value : 1.0000
Neg Pred Value : 0.8958
Prevalence : 0.3857
Detection Rate : 0.3143
Detection Prevalence : 0.3143
Balanced Accuracy : 0.9074
'Positive' Class : 1
compute_aupr(subject_level$filter_150_skeletonized, subject_level$label, 'PR: Segmentations: zero all pixels < 150 and skeletonize')
compute_auroc(subject_level$filter_150_skeletonized, subject_level$label, 'ROC: Segmentations: zero all pixels < 150 and skeletonize')
confusion_matrix(subject_level, 'filter_150_skeletonized')Confusion Matrix and Statistics
Reference
Prediction 0 1
0 38 3
1 5 24
Accuracy : 0.8857
95% CI : (0.7872, 0.9493)
No Information Rate : 0.6143
P-Value [Acc > NIR] : 4.352e-07
Kappa : 0.7621
Mcnemar's Test P-Value : 0.7237
Sensitivity : 0.8889
Specificity : 0.8837
Pos Pred Value : 0.8276
Neg Pred Value : 0.9268
Prevalence : 0.3857
Detection Rate : 0.3429
Detection Prevalence : 0.4143
Balanced Accuracy : 0.8863
'Positive' Class : 1
Segmentations: zero all pixels < 200
compute_aupr(subject_level$filter_200, subject_level$label, 'PR: Segmentations: zero all pixels < 200')
compute_auroc(subject_level$filter_200, subject_level$label, 'ROC: Segmentations: zero all pixels < 200')
confusion_matrix(subject_level, 'filter_200')Confusion Matrix and Statistics
Reference
Prediction 0 1
0 34 4
1 9 23
Accuracy : 0.8143
95% CI : (0.7034, 0.8972)
No Information Rate : 0.6143
P-Value [Acc > NIR] : 0.0002607
Kappa : 0.6211
Mcnemar's Test P-Value : 0.2672575
Sensitivity : 0.8519
Specificity : 0.7907
Pos Pred Value : 0.7187
Neg Pred Value : 0.8947
Prevalence : 0.3857
Detection Rate : 0.3286
Detection Prevalence : 0.4571
Balanced Accuracy : 0.8213
'Positive' Class : 1
compute_aupr(subject_level$filter_200_binarized, subject_level$label, 'PR: Segmentations: zero all pixels < 200 and binarize')
compute_auroc(subject_level$filter_200_binarized, subject_level$label, 'ROC: Segmentations: zero all pixels < 200 and binarize')
confusion_matrix(subject_level, 'filter_200_binarized')Confusion Matrix and Statistics
Reference
Prediction 0 1
0 40 12
1 3 15
Accuracy : 0.7857
95% CI : (0.6713, 0.8748)
No Information Rate : 0.6143
P-Value [Acc > NIR] : 0.001729
Kappa : 0.5179
Mcnemar's Test P-Value : 0.038867
Sensitivity : 0.5556
Specificity : 0.9302
Pos Pred Value : 0.8333
Neg Pred Value : 0.7692
Prevalence : 0.3857
Detection Rate : 0.2143
Detection Prevalence : 0.2571
Balanced Accuracy : 0.7429
'Positive' Class : 1
compute_aupr(subject_level$filter_200_skeletonized, subject_level$label, 'PR: Segmentations: zero all pixels < 200 and skeletonize')
compute_auroc(subject_level$filter_200_skeletonized, subject_level$label, 'ROC: Segmentations: zero all pixels < 200 and skeletonize')
confusion_matrix(subject_level, 'filter_200_skeletonized')Confusion Matrix and Statistics
Reference
Prediction 0 1
0 43 25
1 0 2
Accuracy : 0.6429
95% CI : (0.5193, 0.7539)
No Information Rate : 0.6143
P-Value [Acc > NIR] : 0.3595
Kappa : 0.0895
Mcnemar's Test P-Value : 1.587e-06
Sensitivity : 0.07407
Specificity : 1.00000
Pos Pred Value : 1.00000
Neg Pred Value : 0.63235
Prevalence : 0.38571
Detection Rate : 0.02857
Detection Prevalence : 0.02857
Balanced Accuracy : 0.53704
'Positive' Class : 1
Segmentations: zero all pixels < 210
compute_aupr(subject_level$filter_210, subject_level$label, 'PR: Segmentations: zero all pixels < 210')
compute_auroc(subject_level$filter_210, subject_level$label, 'ROC: Segmentations: zero all pixels < 210')
confusion_matrix(subject_level, 'filter_210')Confusion Matrix and Statistics
Reference
Prediction 0 1
0 43 20
1 0 7
Accuracy : 0.7143
95% CI : (0.5938, 0.816)
No Information Rate : 0.6143
P-Value [Acc > NIR] : 0.0532
Kappa : 0.3007
Mcnemar's Test P-Value : 2.152e-05
Sensitivity : 0.2593
Specificity : 1.0000
Pos Pred Value : 1.0000
Neg Pred Value : 0.6825
Prevalence : 0.3857
Detection Rate : 0.1000
Detection Prevalence : 0.1000
Balanced Accuracy : 0.6296
'Positive' Class : 1
compute_aupr(subject_level$filter_210_binarized, subject_level$label, 'PR: Segmentations: zero all pixels < 210 and binarize')
compute_auroc(subject_level$filter_210_binarized, subject_level$label, 'ROC: Segmentations: zero all pixels < 210 and binarize')
confusion_matrix(subject_level, 'filter_210_binarized')Confusion Matrix and Statistics
Reference
Prediction 0 1
0 39 9
1 4 18
Accuracy : 0.8143
95% CI : (0.7034, 0.8972)
No Information Rate : 0.6143
P-Value [Acc > NIR] : 0.0002607
Kappa : 0.5941
Mcnemar's Test P-Value : 0.2672575
Sensitivity : 0.6667
Specificity : 0.9070
Pos Pred Value : 0.8182
Neg Pred Value : 0.8125
Prevalence : 0.3857
Detection Rate : 0.2571
Detection Prevalence : 0.3143
Balanced Accuracy : 0.7868
'Positive' Class : 1
compute_aupr(subject_level$filter_210_skeletonized, subject_level$label, 'PR: Segmentations: zero all pixels < 210 and skeletonize')
compute_auroc(subject_level$filter_210_skeletonized, subject_level$label, 'ROC: Segmentations: zero all pixels < 210 and skeletonize')
confusion_matrix(subject_level, 'filter_210_skeletonized')Confusion Matrix and Statistics
Reference
Prediction 0 1
0 35 7
1 8 20
Accuracy : 0.7857
95% CI : (0.6713, 0.8748)
No Information Rate : 0.6143
P-Value [Acc > NIR] : 0.001729
Kappa : 0.5509
Mcnemar's Test P-Value : 1.000000
Sensitivity : 0.7407
Specificity : 0.8140
Pos Pred Value : 0.7143
Neg Pred Value : 0.8333
Prevalence : 0.3857
Detection Rate : 0.2857
Detection Prevalence : 0.4000
Balanced Accuracy : 0.7773
'Positive' Class : 1
Segmentations: zero all pixels < 220
compute_aupr(subject_level$filter_220, subject_level$label, 'PR: Segmentations: zero all pixels < 220')
compute_auroc(subject_level$filter_220, subject_level$label, 'ROC: Segmentations: zero all pixels < 220')
confusion_matrix(subject_level, 'filter_220')Confusion Matrix and Statistics
Reference
Prediction 0 1
0 34 12
1 9 15
Accuracy : 0.7
95% CI : (0.5787, 0.8038)
No Information Rate : 0.6143
P-Value [Acc > NIR] : 0.08691
Kappa : 0.3536
Mcnemar's Test P-Value : 0.66252
Sensitivity : 0.5556
Specificity : 0.7907
Pos Pred Value : 0.6250
Neg Pred Value : 0.7391
Prevalence : 0.3857
Detection Rate : 0.2143
Detection Prevalence : 0.3429
Balanced Accuracy : 0.6731
'Positive' Class : 1
compute_aupr(subject_level$filter_220_binarized, subject_level$label, 'PR: Segmentations: zero all pixels < 220 and binarize')
compute_auroc(subject_level$filter_220_binarized, subject_level$label, 'ROC: Segmentations: zero all pixels < 220 and binarize')
confusion_matrix(subject_level, 'filter_220_binarized')Confusion Matrix and Statistics
Reference
Prediction 0 1
0 31 7
1 12 20
Accuracy : 0.7286
95% CI : (0.609, 0.828)
No Information Rate : 0.6143
P-Value [Acc > NIR] : 0.03066
Kappa : 0.4463
Mcnemar's Test P-Value : 0.35880
Sensitivity : 0.7407
Specificity : 0.7209
Pos Pred Value : 0.6250
Neg Pred Value : 0.8158
Prevalence : 0.3857
Detection Rate : 0.2857
Detection Prevalence : 0.4571
Balanced Accuracy : 0.7308
'Positive' Class : 1
compute_aupr(subject_level$filter_220_skeletonized, subject_level$label, 'PR: Segmentations: zero all pixels < 220 and skeletonize')
compute_auroc(subject_level$filter_220_skeletonized, subject_level$label, 'ROC: Segmentations: zero all pixels < 220 and skeletonize')
confusion_matrix(subject_level, 'filter_220_skeletonized')Warning in confusionMatrix.default(preds, df$label, positive = "1") :
Levels are not in the same order for reference and data. Refactoring data to match.Confusion Matrix and Statistics
Reference
Prediction 0 1
0 43 27
1 0 0
Accuracy : 0.6143
95% CI : (0.4903, 0.7283)
No Information Rate : 0.6143
P-Value [Acc > NIR] : 0.5525
Kappa : 0
Mcnemar's Test P-Value : 5.624e-07
Sensitivity : 0.0000
Specificity : 1.0000
Pos Pred Value : NaN
Neg Pred Value : 0.6143
Prevalence : 0.3857
Detection Rate : 0.0000
Detection Prevalence : 0.0000
Balanced Accuracy : 0.5000
'Positive' Class : 1
Segmentations: zero all pixels < 230
compute_aupr(subject_level$filter_230, subject_level$label, 'PR: Segmentations: zero all pixels < 230')
compute_auroc(subject_level$filter_230, subject_level$label, 'ROC: Segmentations: zero all pixels < 230')
confusion_matrix(subject_level, 'filter_230')Confusion Matrix and Statistics
Reference
Prediction 0 1
0 26 6
1 17 21
Accuracy : 0.6714
95% CI : (0.5488, 0.7791)
No Information Rate : 0.6143
P-Value [Acc > NIR] : 0.19581
Kappa : 0.3555
Mcnemar's Test P-Value : 0.03706
Sensitivity : 0.7778
Specificity : 0.6047
Pos Pred Value : 0.5526
Neg Pred Value : 0.8125
Prevalence : 0.3857
Detection Rate : 0.3000
Detection Prevalence : 0.5429
Balanced Accuracy : 0.6912
'Positive' Class : 1
compute_aupr(subject_level$filter_230_binarized, subject_level$label, 'PR: Segmentations: zero all pixels < 230 and binarize')
compute_auroc(subject_level$filter_230_binarized, subject_level$label, 'ROC: Segmentations: zero all pixels < 230 and binarize')
confusion_matrix(subject_level, 'filter_230_binarized')Confusion Matrix and Statistics
Reference
Prediction 0 1
0 10 1
1 33 26
Accuracy : 0.5143
95% CI : (0.3917, 0.6356)
No Information Rate : 0.6143
P-Value [Acc > NIR] : 0.966
Kappa : 0.1602
Mcnemar's Test P-Value : 1.058e-07
Sensitivity : 0.9630
Specificity : 0.2326
Pos Pred Value : 0.4407
Neg Pred Value : 0.9091
Prevalence : 0.3857
Detection Rate : 0.3714
Detection Prevalence : 0.8429
Balanced Accuracy : 0.5978
'Positive' Class : 1
compute_aupr(subject_level$filter_230_skeletonized, subject_level$label, 'PR: Segmentations: zero all pixels < 230 and skeletonize')
compute_auroc(subject_level$filter_230_skeletonized, subject_level$label, 'ROC: Segmentations: zero all pixels < 230 and skeletonize')
confusion_matrix(subject_level, 'filter_230_skeletonized')Confusion Matrix and Statistics
Reference
Prediction 0 1
0 17 2
1 26 25
Accuracy : 0.6
95% CI : (0.4759, 0.7153)
No Information Rate : 0.6143
P-Value [Acc > NIR] : 0.6466
Kappa : 0.2757
Mcnemar's Test P-Value : 1.383e-05
Sensitivity : 0.9259
Specificity : 0.3953
Pos Pred Value : 0.4902
Neg Pred Value : 0.8947
Prevalence : 0.3857
Detection Rate : 0.3571
Detection Prevalence : 0.7286
Balanced Accuracy : 0.6606
'Positive' Class : 1
Segmentations: zero all pixels < 240
compute_aupr(subject_level$filter_240, subject_level$label, 'PR: Segmentations: zero all pixels < 240')
compute_auroc(subject_level$filter_240, subject_level$label, 'ROC: Segmentations: zero all pixels < 240')
confusion_matrix(subject_level, 'filter_240')Confusion Matrix and Statistics
Reference
Prediction 0 1
0 23 9
1 20 18
Accuracy : 0.5857
95% CI : (0.4617, 0.7023)
No Information Rate : 0.6143
P-Value [Acc > NIR] : 0.73214
Kappa : 0.1873
Mcnemar's Test P-Value : 0.06332
Sensitivity : 0.6667
Specificity : 0.5349
Pos Pred Value : 0.4737
Neg Pred Value : 0.7187
Prevalence : 0.3857
Detection Rate : 0.2571
Detection Prevalence : 0.5429
Balanced Accuracy : 0.6008
'Positive' Class : 1
compute_aupr(subject_level$filter_240_binarized, subject_level$label, 'PR: Segmentations: zero all pixels < 240 and binarize')
compute_auroc(subject_level$filter_240_binarized, subject_level$label, 'ROC: Segmentations: zero all pixels < 240 and binarize')
confusion_matrix(subject_level, 'filter_240_binarized')Confusion Matrix and Statistics
Reference
Prediction 0 1
0 12 1
1 31 26
Accuracy : 0.5429
95% CI : (0.4194, 0.6626)
No Information Rate : 0.6143
P-Value [Acc > NIR] : 0.9106
Kappa : 0.2006
Mcnemar's Test P-Value : 2.951e-07
Sensitivity : 0.9630
Specificity : 0.2791
Pos Pred Value : 0.4561
Neg Pred Value : 0.9231
Prevalence : 0.3857
Detection Rate : 0.3714
Detection Prevalence : 0.8143
Balanced Accuracy : 0.6210
'Positive' Class : 1
compute_aupr(subject_level$filter_240_skeletonized, subject_level$label, 'PR: Segmentations: zero all pixels < 240 and skeletonize')
compute_auroc(subject_level$filter_240_skeletonized, subject_level$label, 'ROC: Segmentations: zero all pixels < 240 and skeletonize')
confusion_matrix(subject_level, 'filter_240_skeletonized')Confusion Matrix and Statistics
Reference
Prediction 0 1
0 14 2
1 29 25
Accuracy : 0.5571
95% CI : (0.4334, 0.6759)
No Information Rate : 0.6143
P-Value [Acc > NIR] : 0.8651
Kappa : 0.2121
Mcnemar's Test P-Value : 3.016e-06
Sensitivity : 0.9259
Specificity : 0.3256
Pos Pred Value : 0.4630
Neg Pred Value : 0.8750
Prevalence : 0.3857
Detection Rate : 0.3571
Detection Prevalence : 0.7714
Balanced Accuracy : 0.6258
'Positive' Class : 1
Segmentations: zero all pixels < 250
compute_aupr(subject_level$filter_250, subject_level$label, 'PR: Segmentations: zero all pixels < 250')
compute_auroc(subject_level$filter_250, subject_level$label, 'ROC: Segmentations: zero all pixels < 250')
confusion_matrix(subject_level, 'filter_250')Confusion Matrix and Statistics
Reference
Prediction 0 1
0 1 0
1 42 27
Accuracy : 0.4
95% CI : (0.2847, 0.5241)
No Information Rate : 0.6143
P-Value [Acc > NIR] : 0.9999
Kappa : 0.018
Mcnemar's Test P-Value : 2.509e-10
Sensitivity : 1.00000
Specificity : 0.02326
Pos Pred Value : 0.39130
Neg Pred Value : 1.00000
Prevalence : 0.38571
Detection Rate : 0.38571
Detection Prevalence : 0.98571
Balanced Accuracy : 0.51163
'Positive' Class : 1
compute_aupr(subject_level$filter_250_binarized, subject_level$label, 'PR: Segmentations: zero all pixels < 250 and binarize')
compute_auroc(subject_level$filter_250_binarized, subject_level$label, 'ROC: Segmentations: zero all pixels < 250 and binarize')
confusion_matrix(subject_level, 'filter_250_binarized')Confusion Matrix and Statistics
Reference
Prediction 0 1
0 1 0
1 42 27
Accuracy : 0.4
95% CI : (0.2847, 0.5241)
No Information Rate : 0.6143
P-Value [Acc > NIR] : 0.9999
Kappa : 0.018
Mcnemar's Test P-Value : 2.509e-10
Sensitivity : 1.00000
Specificity : 0.02326
Pos Pred Value : 0.39130
Neg Pred Value : 1.00000
Prevalence : 0.38571
Detection Rate : 0.38571
Detection Prevalence : 0.98571
Balanced Accuracy : 0.51163
'Positive' Class : 1
compute_aupr(subject_level$filter_250_skeletonized, subject_level$label, 'PR: Segmentations: zero all pixels < 250 and skeletonize')
compute_auroc(subject_level$filter_250_skeletonized, subject_level$label, 'ROC: Segmentations: zero all pixels < 250 and skeletonize')
confusion_matrix(subject_level, 'filter_250_skeletonized')Confusion Matrix and Statistics
Reference
Prediction 0 1
0 1 0
1 42 27
Accuracy : 0.4
95% CI : (0.2847, 0.5241)
No Information Rate : 0.6143
P-Value [Acc > NIR] : 0.9999
Kappa : 0.018
Mcnemar's Test P-Value : 2.509e-10
Sensitivity : 1.00000
Specificity : 0.02326
Pos Pred Value : 0.39130
Neg Pred Value : 1.00000
Prevalence : 0.38571
Detection Rate : 0.38571
Detection Prevalence : 0.98571
Balanced Accuracy : 0.51163
'Positive' Class : 1
Segmentations: zero all pixels < 257
compute_aupr(subject_level$filter_257, subject_level$label, 'PR: Segmentations: zero all pixels < 257')
compute_auroc(subject_level$filter_257, subject_level$label, 'ROC: Segmentations: zero all pixels < 257')
confusion_matrix(subject_level, 'filter_257')Warning in confusionMatrix.default(preds, df$label, positive = "1") :
Levels are not in the same order for reference and data. Refactoring data to match.Confusion Matrix and Statistics
Reference
Prediction 0 1
0 43 27
1 0 0
Accuracy : 0.6143
95% CI : (0.4903, 0.7283)
No Information Rate : 0.6143
P-Value [Acc > NIR] : 0.5525
Kappa : 0
Mcnemar's Test P-Value : 5.624e-07
Sensitivity : 0.0000
Specificity : 1.0000
Pos Pred Value : NaN
Neg Pred Value : 0.6143
Prevalence : 0.3857
Detection Rate : 0.0000
Detection Prevalence : 0.0000
Balanced Accuracy : 0.5000
'Positive' Class : 1
compute_aupr(subject_level$filter_257_binarized, subject_level$label, 'PR: Segmentations: zero all pixels < 257 and binarize')
compute_auroc(subject_level$filter_257_binarized, subject_level$label, 'ROC: Segmentations: zero all pixels < 257 and binarize')
confusion_matrix(subject_level, 'filter_257_binarized')Warning in confusionMatrix.default(preds, df$label, positive = "1") :
Levels are not in the same order for reference and data. Refactoring data to match.Confusion Matrix and Statistics
Reference
Prediction 0 1
0 43 27
1 0 0
Accuracy : 0.6143
95% CI : (0.4903, 0.7283)
No Information Rate : 0.6143
P-Value [Acc > NIR] : 0.5525
Kappa : 0
Mcnemar's Test P-Value : 5.624e-07
Sensitivity : 0.0000
Specificity : 1.0000
Pos Pred Value : NaN
Neg Pred Value : 0.6143
Prevalence : 0.3857
Detection Rate : 0.0000
Detection Prevalence : 0.0000
Balanced Accuracy : 0.5000
'Positive' Class : 1
compute_aupr(subject_level$filter_257_skeletonized, subject_level$label, 'PR: Segmentations: zero all pixels < 257 and skeletonize')
compute_auroc(subject_level$filter_257_skeletonized, subject_level$label, 'ROC: Segmentations: zero all pixels < 257 and skeletonize')
confusion_matrix(subject_level, 'filter_257_skeletonized')Warning in confusionMatrix.default(preds, df$label, positive = "1") :
Levels are not in the same order for reference and data. Refactoring data to match.Confusion Matrix and Statistics
Reference
Prediction 0 1
0 43 27
1 0 0
Accuracy : 0.6143
95% CI : (0.4903, 0.7283)
No Information Rate : 0.6143
P-Value [Acc > NIR] : 0.5525
Kappa : 0
Mcnemar's Test P-Value : 5.624e-07
Sensitivity : 0.0000
Specificity : 1.0000
Pos Pred Value : NaN
Neg Pred Value : 0.6143
Prevalence : 0.3857
Detection Rate : 0.0000
Detection Prevalence : 0.0000
Balanced Accuracy : 0.5000
'Positive' Class : 1
Segmentations: zero all pixels > 10
compute_aupr(subject_level$filter_10, subject_level$label, 'PR: Segmentations: zero all pixels < 10')
compute_auroc(subject_level$filter_10, subject_level$label, 'ROC: Segmentations: zero all pixels < 10')
confusion_matrix(subject_level, 'filter_10')Confusion Matrix and Statistics
Reference
Prediction 0 1
0 41 8
1 2 19
Accuracy : 0.8571
95% CI : (0.7529, 0.9293)
No Information Rate : 0.6143
P-Value [Acc > NIR] : 7.748e-06
Kappa : 0.6855
Mcnemar's Test P-Value : 0.1138
Sensitivity : 0.7037
Specificity : 0.9535
Pos Pred Value : 0.9048
Neg Pred Value : 0.8367
Prevalence : 0.3857
Detection Rate : 0.2714
Detection Prevalence : 0.3000
Balanced Accuracy : 0.8286
'Positive' Class : 1
compute_aupr(subject_level$filter_10_binarized, subject_level$label, 'PR: Segmentations: zero all pixels < 10 and binarize')
compute_auroc(subject_level$filter_10_binarized, subject_level$label, 'ROC: Segmentations: zero all pixels < 10 and binarize')
confusion_matrix(subject_level, 'filter_10_binarized')Confusion Matrix and Statistics
Reference
Prediction 0 1
0 34 0
1 9 27
Accuracy : 0.8714
95% CI : (0.7699, 0.9395)
No Information Rate : 0.6143
P-Value [Acc > NIR] : 1.949e-06
Kappa : 0.7445
Mcnemar's Test P-Value : 0.007661
Sensitivity : 1.0000
Specificity : 0.7907
Pos Pred Value : 0.7500
Neg Pred Value : 1.0000
Prevalence : 0.3857
Detection Rate : 0.3857
Detection Prevalence : 0.5143
Balanced Accuracy : 0.8953
'Positive' Class : 1
compute_aupr(subject_level$filter_10_skeletonized, subject_level$label, 'PR: Segmentations: zero all pixels < 10 and skeletonize')
compute_auroc(subject_level$filter_10_skeletonized, subject_level$label, 'ROC: Segmentations: zero all pixels < 10 and skeletonize')
confusion_matrix(subject_level, 'filter_10_skeletonized')Confusion Matrix and Statistics
Reference
Prediction 0 1
0 39 0
1 4 27
Accuracy : 0.9429
95% CI : (0.8601, 0.9842)
No Information Rate : 0.6143
P-Value [Acc > NIR] : 2.41e-10
Kappa : 0.8826
Mcnemar's Test P-Value : 0.1336
Sensitivity : 1.0000
Specificity : 0.9070
Pos Pred Value : 0.8710
Neg Pred Value : 1.0000
Prevalence : 0.3857
Detection Rate : 0.3857
Detection Prevalence : 0.4429
Balanced Accuracy : 0.9535
'Positive' Class : 1
Segmentations: zero all pixels < 75 and > 150
compute_aupr(subject_level$filter_75, subject_level$label, 'PR: Segmentations: zero all pixels < 75 and > 150')
compute_auroc(subject_level$filter_75, subject_level$label, 'ROC: Segmentations: zero all pixels < 75 and > 150')
confusion_matrix(subject_level, 'filter_75')Confusion Matrix and Statistics
Reference
Prediction 0 1
0 36 1
1 7 26
Accuracy : 0.8857
95% CI : (0.7872, 0.9493)
No Information Rate : 0.6143
P-Value [Acc > NIR] : 4.352e-07
Kappa : 0.7684
Mcnemar's Test P-Value : 0.0771
Sensitivity : 0.9630
Specificity : 0.8372
Pos Pred Value : 0.7879
Neg Pred Value : 0.9730
Prevalence : 0.3857
Detection Rate : 0.3714
Detection Prevalence : 0.4714
Balanced Accuracy : 0.9001
'Positive' Class : 1
compute_aupr(subject_level$filter_75_binarized, subject_level$label, 'PR: Segmentations: zero all pixels < 75 and > 150 and binarize')
compute_auroc(subject_level$filter_75_binarized, subject_level$label, 'ROC: Segmentations: zero all pixels < 75 and > 150 and binarize')
confusion_matrix(subject_level, 'filter_75_binarized')Confusion Matrix and Statistics
Reference
Prediction 0 1
0 33 0
1 10 27
Accuracy : 0.8571
95% CI : (0.7529, 0.9293)
No Information Rate : 0.6143
P-Value [Acc > NIR] : 7.748e-06
Kappa : 0.718
Mcnemar's Test P-Value : 0.004427
Sensitivity : 1.0000
Specificity : 0.7674
Pos Pred Value : 0.7297
Neg Pred Value : 1.0000
Prevalence : 0.3857
Detection Rate : 0.3857
Detection Prevalence : 0.5286
Balanced Accuracy : 0.8837
'Positive' Class : 1
compute_aupr(subject_level$filter_75_skeletonized, subject_level$label, 'PR: Segmentations: zero all pixels < 75 and > 150 and skeletonize')
compute_auroc(subject_level$filter_75_skeletonized, subject_level$label, 'ROC: Segmentations: zero all pixels < 75 and > 150 and skeletonize')
confusion_matrix(subject_level, 'filter_75_skeletonized')Confusion Matrix and Statistics
Reference
Prediction 0 1
0 43 13
1 0 14
Accuracy : 0.8143
95% CI : (0.7034, 0.8972)
No Information Rate : 0.6143
P-Value [Acc > NIR] : 0.0002607
Kappa : 0.5695
Mcnemar's Test P-Value : 0.0008741
Sensitivity : 0.5185
Specificity : 1.0000
Pos Pred Value : 1.0000
Neg Pred Value : 0.7679
Prevalence : 0.3857
Detection Rate : 0.2000
Detection Prevalence : 0.2000
Balanced Accuracy : 0.7593
'Positive' Class : 1
LS0tCnRpdGxlOiAnRXZhbHVhdGUgTW9kZWxzJwphdXRob3I6ICdBdXRob3I6IEFhcm9uIFMgQ295bmVyLCBQaEQnCmRhdGU6ICdMYXN0IHVwZGF0ZTogYHIgU3lzLkRhdGUoKWAnCm91dHB1dDoKICAgIGh0bWxfbm90ZWJvb2s6CiAgICAgICAgdG9jOiB5ZXMKICAgICAgICB0b2NfZmxvYXQ6IHllcwogICAgICAgIHRvY19kZXB0aDogMwotLS0KCgojIFNldHVwCmBgYHtyfQpsaWJyYXJ5KHRpZHl2ZXJzZSkKbGlicmFyeSh0b29scykKbGlicmFyeShjYXJldCkKbGlicmFyeShST0NSKQpsaWJyYXJ5KGNhcmV0KQpgYGAKCgpgYGB7cn0KY29tcHV0ZV9hdXJvYyA8LSBmdW5jdGlvbihwcmVkaWN0aW9ucywgbGFiZWxzLCB0aXRsZSkgewogICAgcHJlZCA8LSBwcmVkaWN0aW9uKHByZWRpY3Rpb25zLCBsYWJlbHMpCiAgICBhdXJvYyA8LSBwZXJmb3JtYW5jZShwcmVkLCBtZWFzdXJlID0gJ2F1YycpCiAgICBhdXJvYyA8LSBhdXJvY0B5LnZhbHVlc1tbMV1dCiAgICBwZXJmIDwtIHBlcmZvcm1hbmNlKHByZWQsICd0cHInLCAnZnByJykKICAgIHBsb3QocGVyZiwKICAgICAgICAgbWFpbiA9IHRpdGxlLAogICAgICAgICB4bGltPWMoMCwxKSwKICAgICAgICAgeWxpbT1jKDAsMSksCiAgICAgICAgIGNvbG9yaXplID0gRkFMU0UpCiAgICBhYmxpbmUoYSA9IDAsCiAgICAgICAgICAgYiA9IDEsCiAgICAgICAgICAgY29sID0gJ3JlZCcsCiAgICAgICAgICAgbHR5ID0gMikKICAgIHRleHQoMC44LAogICAgICAgICAwLjI1LAogICAgICAgICBwYXN0ZSgnQVVST0M6Jywgc3ByaW50ZignJS4zZicsIHJvdW5kKGF1cm9jLCAzKSkpKQp9CmBgYAoKCmBgYHtyfQpjb21wdXRlX2F1cHIgPC0gZnVuY3Rpb24ocHJlZGljdGlvbnMsIGxhYmVscywgdGl0bGUpIHsKICAgIG5fYmxhY2sgPC0gbGVuZ3RoKGxhYmVsc1tsYWJlbHMgPT0gMV0pCiAgICBuX3RvdGFsIDwtIGxlbmd0aChsYWJlbHMpCiAgICBwcmVkIDwtIHByZWRpY3Rpb24ocHJlZGljdGlvbnMsIGxhYmVscykKICAgIGF1cHIgPC0gcGVyZm9ybWFuY2UocHJlZCwgbWVhc3VyZSA9ICdhdWNwcicpCiAgICBhdXByIDwtIGF1cHJAeS52YWx1ZXNbWzFdXQogICAgcGVyZiA8LSBwZXJmb3JtYW5jZShwcmVkLCAncHJlYycsICdyZWMnKQogICAgcGxvdChwZXJmLAogICAgICAgICBtYWluID0gdGl0bGUsCiAgICAgICAgIHhsaW09YygwLDEpLAogICAgICAgICB5bGltPWMoMCwxKSwKICAgICAgICAgY29sb3JpemUgPSBGQUxTRSkKICAgIGFibGluZShhID0gbl9ibGFjay9uX3RvdGFsLAogICAgICAgICAgIGIgPSAwLAogICAgICAgICAgIGNvbCA9ICdyZWQnLAogICAgICAgICAgIGx0eSA9IDIpCiAgICB0ZXh0KDAuMiwKICAgICAgICAgMC4yLAogICAgICAgICBwYXN0ZSgnTnVsbCBBVVBSOicsIHNwcmludGYoJyUuM2YnLCByb3VuZChuX2JsYWNrIC8gbl90b3RhbCwgMykpKSkKICAgIHRleHQoMC44LAogICAgICAgICAwLjIsCiAgICAgICAgIHBhc3RlKCdBVVBSOicsIHNwcmludGYoJyUuM2YnLCByb3VuZChhdXByLCAzKSkpKQp9CmBgYAoKCmBgYHtyfQpjb25mdXNpb25fbWF0cml4IDwtIGZ1bmN0aW9uKGRmLCBjb2x1bW4sIHRocmVzaG9sZD0wLjUpIHsKICAgIHByZWRzIDwtIGFzLmZhY3RvcihpZl9lbHNlKGRmW2NvbHVtbl0gPj0gdGhyZXNob2xkLCAxLCAwKSkKICAgIGNvbmZ1c2lvbk1hdHJpeChwcmVkcywgZGYkbGFiZWwsIHBvc2l0aXZlID0gJzEnKQp9CmBgYAoKCgojIExvYWQgZGF0YQpgYGB7cn0KdGVzdF9kYXRhIDwtIHJlYWRfY3N2KCcuL291dC9kYXRhc2V0cy90ZXN0X2RhdGEuY3N2JykgJT4lCiAgICBzZWxlY3Qoc3ViamVjdF9pZCwgaW1hZ2VfaWQpCgoKaW1hZ2VfbGV2ZWwgPC0gcmVhZF9jc3YoJy4vb3V0L3Byb2JhYmlsaXRpZXMvcmV0Y2FtX2ZpbHRlcmVkXzAuY3N2JywgY29sX3R5cGVzID0gY29scygpKSAlPiUKICAgIG11dGF0ZShsYWJlbCA9IGFzLmZhY3RvcihpZl9lbHNlKHN0cl9kZXRlY3QoaW1nX2xvYywgJ2JsYWNrJyksIDEsIDApKSwKICAgICAgICAgICBpbWFnZV9pZCA9IGFzLm51bWVyaWMoZmlsZV9wYXRoX3NhbnNfZXh0KGJhc2VuYW1lKGltZ19sb2MpKSkpICU+JQogICAgc2VsZWN0KGltZ19sb2MsIGltYWdlX2lkLCBsYWJlbCwgcmV0Y2FtID0gcHJvYmFiaWxpdHkpICU+JQogICAgYmluZF9jb2xzKHNlbGVjdChyZWFkX2NzdignLi9vdXQvcHJvYmFiaWxpdGllcy9yZXRjYW1fZmlsdGVyZWRfMF9yYW5kb20uY3N2JywgY29sX3R5cGVzID0gY29scygpKSwgcmV0Y2FtX3JhbmRvbSA9IHByb2JhYmlsaXR5KSkgJT4lCiAgICAKICAgIGJpbmRfY29scyhzZWxlY3QocmVhZF9jc3YoJy4vb3V0L3Byb2JhYmlsaXRpZXMvc2VnbWVudGF0aW9uc19maWx0ZXJlZF8wX3JhbmRvbS5jc3YnLCBjb2xfdHlwZXMgPSBjb2xzKCkpLCBmaWx0ZXJfMF9yYW5kb20gPSBwcm9iYWJpbGl0eSkpICU+JQogICAgYmluZF9jb2xzKHNlbGVjdChyZWFkX2NzdignLi9vdXQvcHJvYmFiaWxpdGllcy9zZWdtZW50YXRpb25zX2ZpbHRlcmVkXzAuY3N2JywgY29sX3R5cGVzID0gY29scygpKSwgZmlsdGVyXzAgPSBwcm9iYWJpbGl0eSkpICU+JQogICAgYmluZF9jb2xzKHNlbGVjdChyZWFkX2NzdignLi9vdXQvcHJvYmFiaWxpdGllcy9zZWdtZW50YXRpb25zX2ZpbHRlcmVkXzBfYmluYXJpemVkLmNzdicsIGNvbF90eXBlcyA9IGNvbHMoKSksIGZpbHRlcl8wX2JpbmFyaXplZCA9IHByb2JhYmlsaXR5KSkgJT4lCiAgICBiaW5kX2NvbHMoc2VsZWN0KHJlYWRfY3N2KCcuL291dC9wcm9iYWJpbGl0aWVzL3NlZ21lbnRhdGlvbnNfZmlsdGVyZWRfMF9za2VsZXRvbml6ZWQuY3N2JywgY29sX3R5cGVzID0gY29scygpKSwgZmlsdGVyXzBfc2tlbGV0b25pemVkID0gcHJvYmFiaWxpdHkpKSAlPiUKICAgIAogICAgYmluZF9jb2xzKHNlbGVjdChyZWFkX2NzdignLi9vdXQvcHJvYmFiaWxpdGllcy9zZWdtZW50YXRpb25zX2ZpbHRlcmVkXzUwLmNzdicsIGNvbF90eXBlcyA9IGNvbHMoKSksIGZpbHRlcl81MCA9IHByb2JhYmlsaXR5KSkgJT4lCiAgICBiaW5kX2NvbHMoc2VsZWN0KHJlYWRfY3N2KCcuL291dC9wcm9iYWJpbGl0aWVzL3NlZ21lbnRhdGlvbnNfZmlsdGVyZWRfNTBfYmluYXJpemVkLmNzdicsIGNvbF90eXBlcyA9IGNvbHMoKSksIGZpbHRlcl81MF9iaW5hcml6ZWQgPSBwcm9iYWJpbGl0eSkpICU+JQogICAgYmluZF9jb2xzKHNlbGVjdChyZWFkX2NzdignLi9vdXQvcHJvYmFiaWxpdGllcy9zZWdtZW50YXRpb25zX2ZpbHRlcmVkXzUwX3NrZWxldG9uaXplZC5jc3YnLCBjb2xfdHlwZXMgPSBjb2xzKCkpLCBmaWx0ZXJfNTBfc2tlbGV0b25pemVkID0gcHJvYmFiaWxpdHkpKSAlPiUKCiAgICBiaW5kX2NvbHMoc2VsZWN0KHJlYWRfY3N2KCcuL291dC9wcm9iYWJpbGl0aWVzL3NlZ21lbnRhdGlvbnNfZmlsdGVyZWRfMTAwLmNzdicsIGNvbF90eXBlcyA9IGNvbHMoKSksIGZpbHRlcl8xMDAgPSBwcm9iYWJpbGl0eSkpICU+JQogICAgYmluZF9jb2xzKHNlbGVjdChyZWFkX2NzdignLi9vdXQvcHJvYmFiaWxpdGllcy9zZWdtZW50YXRpb25zX2ZpbHRlcmVkXzEwMF9iaW5hcml6ZWQuY3N2JywgY29sX3R5cGVzID0gY29scygpKSwgZmlsdGVyXzEwMF9iaW5hcml6ZWQgPSBwcm9iYWJpbGl0eSkpICU+JQogICAgYmluZF9jb2xzKHNlbGVjdChyZWFkX2NzdignLi9vdXQvcHJvYmFiaWxpdGllcy9zZWdtZW50YXRpb25zX2ZpbHRlcmVkXzEwMF9za2VsZXRvbml6ZWQuY3N2JywgY29sX3R5cGVzID0gY29scygpKSwgZmlsdGVyXzEwMF9za2VsZXRvbml6ZWQgPSBwcm9iYWJpbGl0eSkpICU+JQoKICAgIGJpbmRfY29scyhzZWxlY3QocmVhZF9jc3YoJy4vb3V0L3Byb2JhYmlsaXRpZXMvc2VnbWVudGF0aW9uc19maWx0ZXJlZF8xNTAuY3N2JywgY29sX3R5cGVzID0gY29scygpKSwgZmlsdGVyXzE1MCA9IHByb2JhYmlsaXR5KSkgJT4lCiAgICBiaW5kX2NvbHMoc2VsZWN0KHJlYWRfY3N2KCcuL291dC9wcm9iYWJpbGl0aWVzL3NlZ21lbnRhdGlvbnNfZmlsdGVyZWRfMTUwX2JpbmFyaXplZC5jc3YnLCBjb2xfdHlwZXMgPSBjb2xzKCkpLCBmaWx0ZXJfMTUwX2JpbmFyaXplZCA9IHByb2JhYmlsaXR5KSkgJT4lCiAgICBiaW5kX2NvbHMoc2VsZWN0KHJlYWRfY3N2KCcuL291dC9wcm9iYWJpbGl0aWVzL3NlZ21lbnRhdGlvbnNfZmlsdGVyZWRfMTUwX3NrZWxldG9uaXplZC5jc3YnLCBjb2xfdHlwZXMgPSBjb2xzKCkpLCBmaWx0ZXJfMTUwX3NrZWxldG9uaXplZCA9IHByb2JhYmlsaXR5KSkgJT4lCgogICAgYmluZF9jb2xzKHNlbGVjdChyZWFkX2NzdignLi9vdXQvcHJvYmFiaWxpdGllcy9zZWdtZW50YXRpb25zX2ZpbHRlcmVkXzIwMC5jc3YnLCBjb2xfdHlwZXMgPSBjb2xzKCkpLCBmaWx0ZXJfMjAwID0gcHJvYmFiaWxpdHkpKSAlPiUKICAgIGJpbmRfY29scyhzZWxlY3QocmVhZF9jc3YoJy4vb3V0L3Byb2JhYmlsaXRpZXMvc2VnbWVudGF0aW9uc19maWx0ZXJlZF8yMDBfYmluYXJpemVkLmNzdicsIGNvbF90eXBlcyA9IGNvbHMoKSksIGZpbHRlcl8yMDBfYmluYXJpemVkID0gcHJvYmFiaWxpdHkpKSAlPiUKICAgIGJpbmRfY29scyhzZWxlY3QocmVhZF9jc3YoJy4vb3V0L3Byb2JhYmlsaXRpZXMvc2VnbWVudGF0aW9uc19maWx0ZXJlZF8yMDBfc2tlbGV0b25pemVkLmNzdicsIGNvbF90eXBlcyA9IGNvbHMoKSksIGZpbHRlcl8yMDBfc2tlbGV0b25pemVkID0gcHJvYmFiaWxpdHkpKSAlPiUKCiAgICBiaW5kX2NvbHMoc2VsZWN0KHJlYWRfY3N2KCcuL291dC9wcm9iYWJpbGl0aWVzL3NlZ21lbnRhdGlvbnNfZmlsdGVyZWRfMjEwLmNzdicsIGNvbF90eXBlcyA9IGNvbHMoKSksIGZpbHRlcl8yMTAgPSBwcm9iYWJpbGl0eSkpICU+JQogICAgYmluZF9jb2xzKHNlbGVjdChyZWFkX2NzdignLi9vdXQvcHJvYmFiaWxpdGllcy9zZWdtZW50YXRpb25zX2ZpbHRlcmVkXzIxMF9iaW5hcml6ZWQuY3N2JywgY29sX3R5cGVzID0gY29scygpKSwgZmlsdGVyXzIxMF9iaW5hcml6ZWQgPSBwcm9iYWJpbGl0eSkpICU+JQogICAgYmluZF9jb2xzKHNlbGVjdChyZWFkX2NzdignLi9vdXQvcHJvYmFiaWxpdGllcy9zZWdtZW50YXRpb25zX2ZpbHRlcmVkXzIxMF9za2VsZXRvbml6ZWQuY3N2JywgY29sX3R5cGVzID0gY29scygpKSwgZmlsdGVyXzIxMF9za2VsZXRvbml6ZWQgPSBwcm9iYWJpbGl0eSkpICU+JQoKICAgIGJpbmRfY29scyhzZWxlY3QocmVhZF9jc3YoJy4vb3V0L3Byb2JhYmlsaXRpZXMvc2VnbWVudGF0aW9uc19maWx0ZXJlZF8yMjAuY3N2JywgY29sX3R5cGVzID0gY29scygpKSwgZmlsdGVyXzIyMCA9IHByb2JhYmlsaXR5KSkgJT4lCiAgICBiaW5kX2NvbHMoc2VsZWN0KHJlYWRfY3N2KCcuL291dC9wcm9iYWJpbGl0aWVzL3NlZ21lbnRhdGlvbnNfZmlsdGVyZWRfMjIwX2JpbmFyaXplZC5jc3YnLCBjb2xfdHlwZXMgPSBjb2xzKCkpLCBmaWx0ZXJfMjIwX2JpbmFyaXplZCA9IHByb2JhYmlsaXR5KSkgJT4lCiAgICBiaW5kX2NvbHMoc2VsZWN0KHJlYWRfY3N2KCcuL291dC9wcm9iYWJpbGl0aWVzL3NlZ21lbnRhdGlvbnNfZmlsdGVyZWRfMjIwX3NrZWxldG9uaXplZC5jc3YnLCBjb2xfdHlwZXMgPSBjb2xzKCkpLCBmaWx0ZXJfMjIwX3NrZWxldG9uaXplZCA9IHByb2JhYmlsaXR5KSkgJT4lCgogICAgYmluZF9jb2xzKHNlbGVjdChyZWFkX2NzdignLi9vdXQvcHJvYmFiaWxpdGllcy9zZWdtZW50YXRpb25zX2ZpbHRlcmVkXzIzMC5jc3YnLCBjb2xfdHlwZXMgPSBjb2xzKCkpLCBmaWx0ZXJfMjMwID0gcHJvYmFiaWxpdHkpKSAlPiUKICAgIGJpbmRfY29scyhzZWxlY3QocmVhZF9jc3YoJy4vb3V0L3Byb2JhYmlsaXRpZXMvc2VnbWVudGF0aW9uc19maWx0ZXJlZF8yMzBfYmluYXJpemVkLmNzdicsIGNvbF90eXBlcyA9IGNvbHMoKSksIGZpbHRlcl8yMzBfYmluYXJpemVkID0gcHJvYmFiaWxpdHkpKSAlPiUKICAgIGJpbmRfY29scyhzZWxlY3QocmVhZF9jc3YoJy4vb3V0L3Byb2JhYmlsaXRpZXMvc2VnbWVudGF0aW9uc19maWx0ZXJlZF8yMzBfc2tlbGV0b25pemVkLmNzdicsIGNvbF90eXBlcyA9IGNvbHMoKSksIGZpbHRlcl8yMzBfc2tlbGV0b25pemVkID0gcHJvYmFiaWxpdHkpKSAlPiUKCiAgICBiaW5kX2NvbHMoc2VsZWN0KHJlYWRfY3N2KCcuL291dC9wcm9iYWJpbGl0aWVzL3NlZ21lbnRhdGlvbnNfZmlsdGVyZWRfMjQwLmNzdicsIGNvbF90eXBlcyA9IGNvbHMoKSksIGZpbHRlcl8yNDAgPSBwcm9iYWJpbGl0eSkpICU+JQogICAgYmluZF9jb2xzKHNlbGVjdChyZWFkX2NzdignLi9vdXQvcHJvYmFiaWxpdGllcy9zZWdtZW50YXRpb25zX2ZpbHRlcmVkXzI0MF9iaW5hcml6ZWQuY3N2JywgY29sX3R5cGVzID0gY29scygpKSwgZmlsdGVyXzI0MF9iaW5hcml6ZWQgPSBwcm9iYWJpbGl0eSkpICU+JQogICAgYmluZF9jb2xzKHNlbGVjdChyZWFkX2NzdignLi9vdXQvcHJvYmFiaWxpdGllcy9zZWdtZW50YXRpb25zX2ZpbHRlcmVkXzI0MF9za2VsZXRvbml6ZWQuY3N2JywgY29sX3R5cGVzID0gY29scygpKSwgZmlsdGVyXzI0MF9za2VsZXRvbml6ZWQgPSBwcm9iYWJpbGl0eSkpICU+JQoKICAgIGJpbmRfY29scyhzZWxlY3QocmVhZF9jc3YoJy4vb3V0L3Byb2JhYmlsaXRpZXMvc2VnbWVudGF0aW9uc19maWx0ZXJlZF8yNTAuY3N2JywgY29sX3R5cGVzID0gY29scygpKSwgZmlsdGVyXzI1MCA9IHByb2JhYmlsaXR5KSkgJT4lCiAgICBiaW5kX2NvbHMoc2VsZWN0KHJlYWRfY3N2KCcuL291dC9wcm9iYWJpbGl0aWVzL3NlZ21lbnRhdGlvbnNfZmlsdGVyZWRfMjUwX2JpbmFyaXplZC5jc3YnLCBjb2xfdHlwZXMgPSBjb2xzKCkpLCBmaWx0ZXJfMjUwX2JpbmFyaXplZCA9IHByb2JhYmlsaXR5KSkgJT4lCiAgICBiaW5kX2NvbHMoc2VsZWN0KHJlYWRfY3N2KCcuL291dC9wcm9iYWJpbGl0aWVzL3NlZ21lbnRhdGlvbnNfZmlsdGVyZWRfMjUwX3NrZWxldG9uaXplZC5jc3YnLCBjb2xfdHlwZXMgPSBjb2xzKCkpLCBmaWx0ZXJfMjUwX3NrZWxldG9uaXplZCA9IHByb2JhYmlsaXR5KSkgJT4lCgogICAgYmluZF9jb2xzKHNlbGVjdChyZWFkX2NzdignLi9vdXQvcHJvYmFiaWxpdGllcy9zZWdtZW50YXRpb25zX2ZpbHRlcmVkXzI1Ny5jc3YnLCBjb2xfdHlwZXMgPSBjb2xzKCkpLCBmaWx0ZXJfMjU3ID0gcHJvYmFiaWxpdHkpKSAlPiUKICAgIGJpbmRfY29scyhzZWxlY3QocmVhZF9jc3YoJy4vb3V0L3Byb2JhYmlsaXRpZXMvc2VnbWVudGF0aW9uc19maWx0ZXJlZF8yNTdfYmluYXJpemVkLmNzdicsIGNvbF90eXBlcyA9IGNvbHMoKSksIGZpbHRlcl8yNTdfYmluYXJpemVkID0gcHJvYmFiaWxpdHkpKSAlPiUKICAgIGJpbmRfY29scyhzZWxlY3QocmVhZF9jc3YoJy4vb3V0L3Byb2JhYmlsaXRpZXMvc2VnbWVudGF0aW9uc19maWx0ZXJlZF8yNTdfc2tlbGV0b25pemVkLmNzdicsIGNvbF90eXBlcyA9IGNvbHMoKSksIGZpbHRlcl8yNTdfc2tlbGV0b25pemVkID0gcHJvYmFiaWxpdHkpKSAlPiUKCiAgICBiaW5kX2NvbHMoc2VsZWN0KHJlYWRfY3N2KCcuL291dC9wcm9iYWJpbGl0aWVzL3NlZ21lbnRhdGlvbnNfZmlsdGVyZWRfMF8xMC5jc3YnLCBjb2xfdHlwZXMgPSBjb2xzKCkpLCBmaWx0ZXJfMTAgPSBwcm9iYWJpbGl0eSkpICU+JQogICAgYmluZF9jb2xzKHNlbGVjdChyZWFkX2NzdignLi9vdXQvcHJvYmFiaWxpdGllcy9zZWdtZW50YXRpb25zX2ZpbHRlcmVkXzBfMTBfYmluYXJpemVkLmNzdicsIGNvbF90eXBlcyA9IGNvbHMoKSksIGZpbHRlcl8xMF9iaW5hcml6ZWQgPSBwcm9iYWJpbGl0eSkpICU+JQogICAgYmluZF9jb2xzKHNlbGVjdChyZWFkX2NzdignLi9vdXQvcHJvYmFiaWxpdGllcy9zZWdtZW50YXRpb25zX2ZpbHRlcmVkXzBfMTBfc2tlbGV0b25pemVkLmNzdicsIGNvbF90eXBlcyA9IGNvbHMoKSksIGZpbHRlcl8xMF9za2VsZXRvbml6ZWQgPSBwcm9iYWJpbGl0eSkpICU+JQoKICAgIGJpbmRfY29scyhzZWxlY3QocmVhZF9jc3YoJy4vb3V0L3Byb2JhYmlsaXRpZXMvc2VnbWVudGF0aW9uc19maWx0ZXJlZF83NV8xNTAuY3N2JywgY29sX3R5cGVzID0gY29scygpKSwgZmlsdGVyXzc1ID0gcHJvYmFiaWxpdHkpKSAlPiUKICAgIGJpbmRfY29scyhzZWxlY3QocmVhZF9jc3YoJy4vb3V0L3Byb2JhYmlsaXRpZXMvc2VnbWVudGF0aW9uc19maWx0ZXJlZF83NV8xNTBfYmluYXJpemVkLmNzdicsIGNvbF90eXBlcyA9IGNvbHMoKSksIGZpbHRlcl83NV9iaW5hcml6ZWQgPSBwcm9iYWJpbGl0eSkpICU+JQogICAgYmluZF9jb2xzKHNlbGVjdChyZWFkX2NzdignLi9vdXQvcHJvYmFiaWxpdGllcy9zZWdtZW50YXRpb25zX2ZpbHRlcmVkXzc1XzE1MF9za2VsZXRvbml6ZWQuY3N2JywgY29sX3R5cGVzID0gY29scygpKSwgZmlsdGVyXzc1X3NrZWxldG9uaXplZCA9IHByb2JhYmlsaXR5KSkgJT4lCiAgICAKICAgIGlubmVyX2pvaW4odGVzdF9kYXRhLCBieSA9ICdpbWFnZV9pZCcpICU+JQogICAgc2VsZWN0KHN1YmplY3RfaWQsIGV2ZXJ5dGhpbmcoKSwgLWltZ19sb2MpCgoKCnN1YmplY3RfbGV2ZWwgPC0gaW1hZ2VfbGV2ZWwgJT4lCiAgICBncm91cF9ieShzdWJqZWN0X2lkKSAlPiUKICAgIG11dGF0ZShhY3Jvc3MoYygtaW1hZ2VfaWQsIC1sYWJlbCksIG1lZGlhbikpICU+JQogICAgdW5ncm91cCgpICU+JQogICAgZGlzdGluY3Qoc3ViamVjdF9pZCwgLmtlZXBfYWxsID0gVFJVRSkKYGBgCgoKIyBJbWFnZS1sZXZlbCBBbmFseXNpcwoKIyMjIFJldENhbSBJbWFnZXMKCmBgYHtyfQpjb21wdXRlX2F1cHIoaW1hZ2VfbGV2ZWwkcmV0Y2FtLCBpbWFnZV9sZXZlbCRsYWJlbCwgJ1BSOiBSYXcgUmV0Q2FtIEltYWdlcycpCmNvbXB1dGVfYXVyb2MoaW1hZ2VfbGV2ZWwkcmV0Y2FtLCBpbWFnZV9sZXZlbCRsYWJlbCwgJ1JPQzogUmF3IFJldENhbSBJbWFnZXMnKQpjb25mdXNpb25fbWF0cml4KGltYWdlX2xldmVsLCAncmV0Y2FtJykKYGBgCgpgYGB7cn0KY29tcHV0ZV9hdXByKGltYWdlX2xldmVsJHJldGNhbV9yYW5kb20sIGltYWdlX2xldmVsJGxhYmVsLCAnUFI6IFJhdyBSZXRDYW0gSW1hZ2VzIC0gU2h1ZmZsZWQgTGFiZWxzJykKY29tcHV0ZV9hdXJvYyhpbWFnZV9sZXZlbCRyZXRjYW1fcmFuZG9tLCBpbWFnZV9sZXZlbCRsYWJlbCwgJ1JPQzogUmF3IFJldENhbSBJbWFnZXMgLSBTaHVmZmxlZCBMYWJlbHMnKQpjb25mdXNpb25fbWF0cml4KGltYWdlX2xldmVsLCAncmV0Y2FtX3JhbmRvbScpCmBgYAoKCgojIyMgU2VnbWVudGF0aW9uczogemVybyBhbGwgcGl4ZWxzIDwgMAoKYGBge3J9CmNvbXB1dGVfYXVwcihpbWFnZV9sZXZlbCRmaWx0ZXJfMCwgaW1hZ2VfbGV2ZWwkbGFiZWwsICdQUjogU2VnbWVudGF0aW9uczogemVybyBhbGwgcGl4ZWxzIDwgMCcpCmNvbXB1dGVfYXVyb2MoaW1hZ2VfbGV2ZWwkZmlsdGVyXzAsIGltYWdlX2xldmVsJGxhYmVsLCAnUk9DOiBTZWdtZW50YXRpb25zOiB6ZXJvIGFsbCBwaXhlbHMgPCAwJykKY29uZnVzaW9uX21hdHJpeChpbWFnZV9sZXZlbCwgJ2ZpbHRlcl8wJykKYGBgCgpgYGB7cn0KY29tcHV0ZV9hdXByKGltYWdlX2xldmVsJGZpbHRlcl8wX2JpbmFyaXplZCwgaW1hZ2VfbGV2ZWwkbGFiZWwsICdQUjogU2VnbWVudGF0aW9uczogemVybyBhbGwgcGl4ZWxzIDwgMCBhbmQgYmluYXJpemUnKQpjb21wdXRlX2F1cm9jKGltYWdlX2xldmVsJGZpbHRlcl8wX2JpbmFyaXplZCwgaW1hZ2VfbGV2ZWwkbGFiZWwsICdST0M6IFNlZ21lbnRhdGlvbnM6IHplcm8gYWxsIHBpeGVscyA8IDAgYW5kIGJpbmFyaXplJykKY29uZnVzaW9uX21hdHJpeChpbWFnZV9sZXZlbCwgJ2ZpbHRlcl8wX2JpbmFyaXplZCcpCmBgYAoKYGBge3J9CmNvbXB1dGVfYXVwcihpbWFnZV9sZXZlbCRmaWx0ZXJfMF9za2VsZXRvbml6ZWQsIGltYWdlX2xldmVsJGxhYmVsLCAnUFI6IFNlZ21lbnRhdGlvbnM6IHplcm8gYWxsIHBpeGVscyA8IDAgYW5kIHNrZWxldG9uaXplJykKY29tcHV0ZV9hdXJvYyhpbWFnZV9sZXZlbCRmaWx0ZXJfMF9za2VsZXRvbml6ZWQsIGltYWdlX2xldmVsJGxhYmVsLCAnUk9DOiBTZWdtZW50YXRpb25zOiB6ZXJvIGFsbCBwaXhlbHMgPCAwIGFuZCBza2VsZXRvbml6ZScpCmNvbmZ1c2lvbl9tYXRyaXgoaW1hZ2VfbGV2ZWwsICdmaWx0ZXJfMF9za2VsZXRvbml6ZWQnKQpgYGAKCmBgYHtyfQpjb21wdXRlX2F1cHIoaW1hZ2VfbGV2ZWwkZmlsdGVyXzBfcmFuZG9tLCBpbWFnZV9sZXZlbCRsYWJlbCwgJ1BSOiBTZWdtZW50YXRpb25zOiB6ZXJvIGFsbCBwaXhlbHMgPCAwIC0gU2h1ZmZsZWQgTGFiZWxzJykKY29tcHV0ZV9hdXJvYyhpbWFnZV9sZXZlbCRmaWx0ZXJfMF9yYW5kb20sIGltYWdlX2xldmVsJGxhYmVsLCAnUk9DOiBTZWdtZW50YXRpb25zOiB6ZXJvIGFsbCBwaXhlbHMgPCAwIC0gU2h1ZmZsZWQgTGFiZWxzJykKY29uZnVzaW9uX21hdHJpeChpbWFnZV9sZXZlbCwgJ2ZpbHRlcl8wX3JhbmRvbScpCmBgYAoKCiMjIyBTZWdtZW50YXRpb25zOiB6ZXJvIGFsbCBwaXhlbHMgPCA1MAoKYGBge3J9CmNvbXB1dGVfYXVwcihpbWFnZV9sZXZlbCRmaWx0ZXJfNTAsIGltYWdlX2xldmVsJGxhYmVsLCAnUFI6IFNlZ21lbnRhdGlvbnM6IHplcm8gYWxsIHBpeGVscyA8IDUwJykKY29tcHV0ZV9hdXJvYyhpbWFnZV9sZXZlbCRmaWx0ZXJfNTAsIGltYWdlX2xldmVsJGxhYmVsLCAnUk9DOiBTZWdtZW50YXRpb25zOiB6ZXJvIGFsbCBwaXhlbHMgPCA1MCcpCmNvbmZ1c2lvbl9tYXRyaXgoaW1hZ2VfbGV2ZWwsICdmaWx0ZXJfNTAnKQpgYGAKCmBgYHtyfQpjb21wdXRlX2F1cHIoaW1hZ2VfbGV2ZWwkZmlsdGVyXzUwX2JpbmFyaXplZCwgaW1hZ2VfbGV2ZWwkbGFiZWwsICdQUjogU2VnbWVudGF0aW9uczogemVybyBhbGwgcGl4ZWxzIDwgNTAgYW5kIGJpbmFyaXplJykKY29tcHV0ZV9hdXJvYyhpbWFnZV9sZXZlbCRmaWx0ZXJfNTBfYmluYXJpemVkLCBpbWFnZV9sZXZlbCRsYWJlbCwgJ1JPQzogU2VnbWVudGF0aW9uczogemVybyBhbGwgcGl4ZWxzIDwgNTAgYW5kIGJpbmFyaXplJykKY29uZnVzaW9uX21hdHJpeChpbWFnZV9sZXZlbCwgJ2ZpbHRlcl8wX2JpbmFyaXplZCcpCmBgYAoKYGBge3J9CmNvbXB1dGVfYXVwcihpbWFnZV9sZXZlbCRmaWx0ZXJfNTBfc2tlbGV0b25pemVkLCBpbWFnZV9sZXZlbCRsYWJlbCwgJ1BSOiBTZWdtZW50YXRpb25zOiB6ZXJvIGFsbCBwaXhlbHMgPCA1MCBhbmQgc2tlbGV0b25pemUnKQpjb21wdXRlX2F1cm9jKGltYWdlX2xldmVsJGZpbHRlcl81MF9za2VsZXRvbml6ZWQsIGltYWdlX2xldmVsJGxhYmVsLCAnUk9DOiBTZWdtZW50YXRpb25zOiB6ZXJvIGFsbCBwaXhlbHMgPCA1MCBhbmQgc2tlbGV0b25pemUnKQpjb25mdXNpb25fbWF0cml4KGltYWdlX2xldmVsLCAnZmlsdGVyXzUwX3NrZWxldG9uaXplZCcpCmBgYAoKCgojIyMgU2VnbWVudGF0aW9uczogemVybyBhbGwgcGl4ZWxzIDwgMTAwCgpgYGB7cn0KY29tcHV0ZV9hdXByKGltYWdlX2xldmVsJGZpbHRlcl8xMDAsIGltYWdlX2xldmVsJGxhYmVsLCAnUFI6IFNlZ21lbnRhdGlvbnM6IHplcm8gYWxsIHBpeGVscyA8IDEwMCcpCmNvbXB1dGVfYXVyb2MoaW1hZ2VfbGV2ZWwkZmlsdGVyXzEwMCwgaW1hZ2VfbGV2ZWwkbGFiZWwsICdST0M6IFNlZ21lbnRhdGlvbnM6IHplcm8gYWxsIHBpeGVscyA8IDEwMCcpCmNvbmZ1c2lvbl9tYXRyaXgoaW1hZ2VfbGV2ZWwsICdmaWx0ZXJfMTAwJykKYGBgCgpgYGB7cn0KY29tcHV0ZV9hdXByKGltYWdlX2xldmVsJGZpbHRlcl8xMDBfYmluYXJpemVkLCBpbWFnZV9sZXZlbCRsYWJlbCwgJ1BSOiBTZWdtZW50YXRpb25zOiB6ZXJvIGFsbCBwaXhlbHMgPCAxMDAgYW5kIGJpbmFyaXplJykKY29tcHV0ZV9hdXJvYyhpbWFnZV9sZXZlbCRmaWx0ZXJfMTAwX2JpbmFyaXplZCwgaW1hZ2VfbGV2ZWwkbGFiZWwsICdST0M6IFNlZ21lbnRhdGlvbnM6IHplcm8gYWxsIHBpeGVscyA8IDEwMCBhbmQgYmluYXJpemUnKQpjb25mdXNpb25fbWF0cml4KGltYWdlX2xldmVsLCAnZmlsdGVyXzEwMF9iaW5hcml6ZWQnKQpgYGAKCmBgYHtyfQpjb21wdXRlX2F1cHIoaW1hZ2VfbGV2ZWwkZmlsdGVyXzEwMF9za2VsZXRvbml6ZWQsIGltYWdlX2xldmVsJGxhYmVsLCAnUFI6IFNlZ21lbnRhdGlvbnM6IHplcm8gYWxsIHBpeGVscyA8IDEwMCBhbmQgc2tlbGV0b25pemUnKQpjb21wdXRlX2F1cm9jKGltYWdlX2xldmVsJGZpbHRlcl8xMDBfc2tlbGV0b25pemVkLCBpbWFnZV9sZXZlbCRsYWJlbCwgJ1JPQzogU2VnbWVudGF0aW9uczogemVybyBhbGwgcGl4ZWxzIDwgMTAwIGFuZCBza2VsZXRvbml6ZScpCmNvbmZ1c2lvbl9tYXRyaXgoaW1hZ2VfbGV2ZWwsICdmaWx0ZXJfMTAwX3NrZWxldG9uaXplZCcpCmBgYAoKCgojIyMgU2VnbWVudGF0aW9uczogemVybyBhbGwgcGl4ZWxzIDwgMTUwCgpgYGB7cn0KY29tcHV0ZV9hdXByKGltYWdlX2xldmVsJGZpbHRlcl8xNTAsIGltYWdlX2xldmVsJGxhYmVsLCAnUFI6IFNlZ21lbnRhdGlvbnM6IHplcm8gYWxsIHBpeGVscyA8IDE1MCcpCmNvbXB1dGVfYXVyb2MoaW1hZ2VfbGV2ZWwkZmlsdGVyXzE1MCwgaW1hZ2VfbGV2ZWwkbGFiZWwsICdST0M6IFNlZ21lbnRhdGlvbnM6IHplcm8gYWxsIHBpeGVscyA8IDE1MCcpCmNvbmZ1c2lvbl9tYXRyaXgoaW1hZ2VfbGV2ZWwsICdmaWx0ZXJfMTUwJykKYGBgCgpgYGB7cn0KY29tcHV0ZV9hdXByKGltYWdlX2xldmVsJGZpbHRlcl8xNTBfYmluYXJpemVkLCBpbWFnZV9sZXZlbCRsYWJlbCwgJ1BSOiBTZWdtZW50YXRpb25zOiB6ZXJvIGFsbCBwaXhlbHMgPCAxNTAgYW5kIGJpbmFyaXplJykKY29tcHV0ZV9hdXJvYyhpbWFnZV9sZXZlbCRmaWx0ZXJfMTUwX2JpbmFyaXplZCwgaW1hZ2VfbGV2ZWwkbGFiZWwsICdST0M6IFNlZ21lbnRhdGlvbnM6IHplcm8gYWxsIHBpeGVscyA8IDE1MCBhbmQgYmluYXJpemUnKQpjb25mdXNpb25fbWF0cml4KGltYWdlX2xldmVsLCAnZmlsdGVyXzE1MF9iaW5hcml6ZWQnKQpgYGAKCmBgYHtyfQpjb21wdXRlX2F1cHIoaW1hZ2VfbGV2ZWwkZmlsdGVyXzE1MF9za2VsZXRvbml6ZWQsIGltYWdlX2xldmVsJGxhYmVsLCAnUFI6IFNlZ21lbnRhdGlvbnM6IHplcm8gYWxsIHBpeGVscyA8IDE1MCBhbmQgc2tlbGV0b25pemUnKQpjb21wdXRlX2F1cm9jKGltYWdlX2xldmVsJGZpbHRlcl8xNTBfc2tlbGV0b25pemVkLCBpbWFnZV9sZXZlbCRsYWJlbCwgJ1JPQzogU2VnbWVudGF0aW9uczogemVybyBhbGwgcGl4ZWxzIDwgMTUwIGFuZCBza2VsZXRvbml6ZScpCmNvbmZ1c2lvbl9tYXRyaXgoaW1hZ2VfbGV2ZWwsICdmaWx0ZXJfMTUwX3NrZWxldG9uaXplZCcpCmBgYAoKCgojIyMgU2VnbWVudGF0aW9uczogemVybyBhbGwgcGl4ZWxzIDwgMjAwCgpgYGB7cn0KY29tcHV0ZV9hdXByKGltYWdlX2xldmVsJGZpbHRlcl8yMDAsIGltYWdlX2xldmVsJGxhYmVsLCAnUFI6IFNlZ21lbnRhdGlvbnM6IHplcm8gYWxsIHBpeGVscyA8IDIwMCcpCmNvbXB1dGVfYXVyb2MoaW1hZ2VfbGV2ZWwkZmlsdGVyXzIwMCwgaW1hZ2VfbGV2ZWwkbGFiZWwsICdST0M6IFNlZ21lbnRhdGlvbnM6IHplcm8gYWxsIHBpeGVscyA8IDIwMCcpCmNvbmZ1c2lvbl9tYXRyaXgoaW1hZ2VfbGV2ZWwsICdmaWx0ZXJfMjAwJykKYGBgCgpgYGB7cn0KY29tcHV0ZV9hdXByKGltYWdlX2xldmVsJGZpbHRlcl8yMDBfYmluYXJpemVkLCBpbWFnZV9sZXZlbCRsYWJlbCwgJ1BSOiBTZWdtZW50YXRpb25zOiB6ZXJvIGFsbCBwaXhlbHMgPCAyMDAgYW5kIGJpbmFyaXplJykKY29tcHV0ZV9hdXJvYyhpbWFnZV9sZXZlbCRmaWx0ZXJfMjAwX2JpbmFyaXplZCwgaW1hZ2VfbGV2ZWwkbGFiZWwsICdST0M6IFNlZ21lbnRhdGlvbnM6IHplcm8gYWxsIHBpeGVscyA8IDIwMCBhbmQgYmluYXJpemUnKQpjb25mdXNpb25fbWF0cml4KGltYWdlX2xldmVsLCAnZmlsdGVyXzIwMF9iaW5hcml6ZWQnKQpgYGAKCmBgYHtyfQpjb21wdXRlX2F1cHIoaW1hZ2VfbGV2ZWwkZmlsdGVyXzIwMF9za2VsZXRvbml6ZWQsIGltYWdlX2xldmVsJGxhYmVsLCAnUFI6IFNlZ21lbnRhdGlvbnM6IHplcm8gYWxsIHBpeGVscyA8IDIwMCBhbmQgc2tlbGV0b25pemUnKQpjb21wdXRlX2F1cm9jKGltYWdlX2xldmVsJGZpbHRlcl8yMDBfc2tlbGV0b25pemVkLCBpbWFnZV9sZXZlbCRsYWJlbCwgJ1JPQzogU2VnbWVudGF0aW9uczogemVybyBhbGwgcGl4ZWxzIDwgMjAwIGFuZCBza2VsZXRvbml6ZScpCmNvbmZ1c2lvbl9tYXRyaXgoaW1hZ2VfbGV2ZWwsICdmaWx0ZXJfMjAwX3NrZWxldG9uaXplZCcpCmBgYAoKCgojIyMgU2VnbWVudGF0aW9uczogemVybyBhbGwgcGl4ZWxzIDwgMjEwCgpgYGB7cn0KY29tcHV0ZV9hdXByKGltYWdlX2xldmVsJGZpbHRlcl8yMTAsIGltYWdlX2xldmVsJGxhYmVsLCAnUFI6IFNlZ21lbnRhdGlvbnM6IHplcm8gYWxsIHBpeGVscyA8IDIxMCcpCmNvbXB1dGVfYXVyb2MoaW1hZ2VfbGV2ZWwkZmlsdGVyXzIxMCwgaW1hZ2VfbGV2ZWwkbGFiZWwsICdST0M6IFNlZ21lbnRhdGlvbnM6IHplcm8gYWxsIHBpeGVscyA8IDIxMCcpCmNvbmZ1c2lvbl9tYXRyaXgoaW1hZ2VfbGV2ZWwsICdmaWx0ZXJfMjEwJykKYGBgCgpgYGB7cn0KY29tcHV0ZV9hdXByKGltYWdlX2xldmVsJGZpbHRlcl8yMTBfYmluYXJpemVkLCBpbWFnZV9sZXZlbCRsYWJlbCwgJ1BSOiBTZWdtZW50YXRpb25zOiB6ZXJvIGFsbCBwaXhlbHMgPCAyMTAgYW5kIGJpbmFyaXplJykKY29tcHV0ZV9hdXJvYyhpbWFnZV9sZXZlbCRmaWx0ZXJfMjEwX2JpbmFyaXplZCwgaW1hZ2VfbGV2ZWwkbGFiZWwsICdST0M6IFNlZ21lbnRhdGlvbnM6IHplcm8gYWxsIHBpeGVscyA8IDIxMCBhbmQgYmluYXJpemUnKQpjb25mdXNpb25fbWF0cml4KGltYWdlX2xldmVsLCAnZmlsdGVyXzIxMF9iaW5hcml6ZWQnKQpgYGAKCmBgYHtyfQpjb21wdXRlX2F1cHIoaW1hZ2VfbGV2ZWwkZmlsdGVyXzIxMF9za2VsZXRvbml6ZWQsIGltYWdlX2xldmVsJGxhYmVsLCAnUFI6IFNlZ21lbnRhdGlvbnM6IHplcm8gYWxsIHBpeGVscyA8IDIxMCBhbmQgc2tlbGV0b25pemUnKQpjb21wdXRlX2F1cm9jKGltYWdlX2xldmVsJGZpbHRlcl8yMTBfc2tlbGV0b25pemVkLCBpbWFnZV9sZXZlbCRsYWJlbCwgJ1JPQzogU2VnbWVudGF0aW9uczogemVybyBhbGwgcGl4ZWxzIDwgMjEwIGFuZCBza2VsZXRvbml6ZScpCmNvbmZ1c2lvbl9tYXRyaXgoaW1hZ2VfbGV2ZWwsICdmaWx0ZXJfMjEwX3NrZWxldG9uaXplZCcpCmBgYAoKCgojIyMgU2VnbWVudGF0aW9uczogemVybyBhbGwgcGl4ZWxzIDwgMjIwCgpgYGB7cn0KY29tcHV0ZV9hdXByKGltYWdlX2xldmVsJGZpbHRlcl8yMjAsIGltYWdlX2xldmVsJGxhYmVsLCAnUFI6IFNlZ21lbnRhdGlvbnM6IHplcm8gYWxsIHBpeGVscyA8IDIyMCcpCmNvbXB1dGVfYXVyb2MoaW1hZ2VfbGV2ZWwkZmlsdGVyXzIyMCwgaW1hZ2VfbGV2ZWwkbGFiZWwsICdST0M6IFNlZ21lbnRhdGlvbnM6IHplcm8gYWxsIHBpeGVscyA8IDIyMCcpCmNvbmZ1c2lvbl9tYXRyaXgoaW1hZ2VfbGV2ZWwsICdmaWx0ZXJfMjIwJykKYGBgCgpgYGB7cn0KY29tcHV0ZV9hdXByKGltYWdlX2xldmVsJGZpbHRlcl8yMjBfYmluYXJpemVkLCBpbWFnZV9sZXZlbCRsYWJlbCwgJ1BSOiBTZWdtZW50YXRpb25zOiB6ZXJvIGFsbCBwaXhlbHMgPCAyMjAgYW5kIGJpbmFyaXplJykKY29tcHV0ZV9hdXJvYyhpbWFnZV9sZXZlbCRmaWx0ZXJfMjIwX2JpbmFyaXplZCwgaW1hZ2VfbGV2ZWwkbGFiZWwsICdST0M6IFNlZ21lbnRhdGlvbnM6IHplcm8gYWxsIHBpeGVscyA8IDIyMCBhbmQgYmluYXJpemUnKQpjb25mdXNpb25fbWF0cml4KGltYWdlX2xldmVsLCAnZmlsdGVyXzIyMF9iaW5hcml6ZWQnKQpgYGAKCmBgYHtyfQpjb21wdXRlX2F1cHIoaW1hZ2VfbGV2ZWwkZmlsdGVyXzIyMF9za2VsZXRvbml6ZWQsIGltYWdlX2xldmVsJGxhYmVsLCAnUFI6IFNlZ21lbnRhdGlvbnM6IHplcm8gYWxsIHBpeGVscyA8IDIyMCBhbmQgc2tlbGV0b25pemUnKQpjb21wdXRlX2F1cm9jKGltYWdlX2xldmVsJGZpbHRlcl8yMjBfc2tlbGV0b25pemVkLCBpbWFnZV9sZXZlbCRsYWJlbCwgJ1JPQzogU2VnbWVudGF0aW9uczogemVybyBhbGwgcGl4ZWxzIDwgMjIwIGFuZCBza2VsZXRvbml6ZScpCmNvbmZ1c2lvbl9tYXRyaXgoaW1hZ2VfbGV2ZWwsICdmaWx0ZXJfMjIwX3NrZWxldG9uaXplZCcpCmBgYAoKCgojIyMgU2VnbWVudGF0aW9uczogemVybyBhbGwgcGl4ZWxzIDwgMjMwCgpgYGB7cn0KY29tcHV0ZV9hdXByKGltYWdlX2xldmVsJGZpbHRlcl8yMzAsIGltYWdlX2xldmVsJGxhYmVsLCAnUFI6IFNlZ21lbnRhdGlvbnM6IHplcm8gYWxsIHBpeGVscyA8IDIzMCcpCmNvbXB1dGVfYXVyb2MoaW1hZ2VfbGV2ZWwkZmlsdGVyXzIzMCwgaW1hZ2VfbGV2ZWwkbGFiZWwsICdST0M6IFNlZ21lbnRhdGlvbnM6IHplcm8gYWxsIHBpeGVscyA8IDIzMCcpCmNvbmZ1c2lvbl9tYXRyaXgoaW1hZ2VfbGV2ZWwsICdmaWx0ZXJfMjMwJykKYGBgCgpgYGB7cn0KY29tcHV0ZV9hdXByKGltYWdlX2xldmVsJGZpbHRlcl8yMzBfYmluYXJpemVkLCBpbWFnZV9sZXZlbCRsYWJlbCwgJ1BSOiBTZWdtZW50YXRpb25zOiB6ZXJvIGFsbCBwaXhlbHMgPCAyMzAgYW5kIGJpbmFyaXplJykKY29tcHV0ZV9hdXJvYyhpbWFnZV9sZXZlbCRmaWx0ZXJfMjMwX2JpbmFyaXplZCwgaW1hZ2VfbGV2ZWwkbGFiZWwsICdST0M6IFNlZ21lbnRhdGlvbnM6IHplcm8gYWxsIHBpeGVscyA8IDIzMCBhbmQgYmluYXJpemUnKQpjb25mdXNpb25fbWF0cml4KGltYWdlX2xldmVsLCAnZmlsdGVyXzIzMF9iaW5hcml6ZWQnKQpgYGAKCmBgYHtyfQpjb21wdXRlX2F1cHIoaW1hZ2VfbGV2ZWwkZmlsdGVyXzIzMF9za2VsZXRvbml6ZWQsIGltYWdlX2xldmVsJGxhYmVsLCAnUFI6IFNlZ21lbnRhdGlvbnM6IHplcm8gYWxsIHBpeGVscyA8IDIzMCBhbmQgc2tlbGV0b25pemUnKQpjb21wdXRlX2F1cm9jKGltYWdlX2xldmVsJGZpbHRlcl8yMzBfc2tlbGV0b25pemVkLCBpbWFnZV9sZXZlbCRsYWJlbCwgJ1JPQzogU2VnbWVudGF0aW9uczogemVybyBhbGwgcGl4ZWxzIDwgMjMwIGFuZCBza2VsZXRvbml6ZScpCmNvbmZ1c2lvbl9tYXRyaXgoaW1hZ2VfbGV2ZWwsICdmaWx0ZXJfMjMwX3NrZWxldG9uaXplZCcpCmBgYAoKCgojIyMgU2VnbWVudGF0aW9uczogemVybyBhbGwgcGl4ZWxzIDwgMjQwCgpgYGB7cn0KY29tcHV0ZV9hdXByKGltYWdlX2xldmVsJGZpbHRlcl8yNDAsIGltYWdlX2xldmVsJGxhYmVsLCAnUFI6IFNlZ21lbnRhdGlvbnM6IHplcm8gYWxsIHBpeGVscyA8IDI0MCcpCmNvbXB1dGVfYXVyb2MoaW1hZ2VfbGV2ZWwkZmlsdGVyXzI0MCwgaW1hZ2VfbGV2ZWwkbGFiZWwsICdST0M6IFNlZ21lbnRhdGlvbnM6IHplcm8gYWxsIHBpeGVscyA8IDI0MCcpCmNvbmZ1c2lvbl9tYXRyaXgoaW1hZ2VfbGV2ZWwsICdmaWx0ZXJfMjQwJykKYGBgCgpgYGB7cn0KY29tcHV0ZV9hdXByKGltYWdlX2xldmVsJGZpbHRlcl8yNDBfYmluYXJpemVkLCBpbWFnZV9sZXZlbCRsYWJlbCwgJ1BSOiBTZWdtZW50YXRpb25zOiB6ZXJvIGFsbCBwaXhlbHMgPCAyNDAgYW5kIGJpbmFyaXplJykKY29tcHV0ZV9hdXJvYyhpbWFnZV9sZXZlbCRmaWx0ZXJfMjQwX2JpbmFyaXplZCwgaW1hZ2VfbGV2ZWwkbGFiZWwsICdST0M6IFNlZ21lbnRhdGlvbnM6IHplcm8gYWxsIHBpeGVscyA8IDI0MCBhbmQgYmluYXJpemUnKQpjb25mdXNpb25fbWF0cml4KGltYWdlX2xldmVsLCAnZmlsdGVyXzI0MF9iaW5hcml6ZWQnKQpgYGAKCmBgYHtyfQpjb21wdXRlX2F1cHIoaW1hZ2VfbGV2ZWwkZmlsdGVyXzI0MF9za2VsZXRvbml6ZWQsIGltYWdlX2xldmVsJGxhYmVsLCAnUFI6IFNlZ21lbnRhdGlvbnM6IHplcm8gYWxsIHBpeGVscyA8IDI0MCBhbmQgc2tlbGV0b25pemUnKQpjb21wdXRlX2F1cm9jKGltYWdlX2xldmVsJGZpbHRlcl8yNDBfc2tlbGV0b25pemVkLCBpbWFnZV9sZXZlbCRsYWJlbCwgJ1JPQzogU2VnbWVudGF0aW9uczogemVybyBhbGwgcGl4ZWxzIDwgMjQwIGFuZCBza2VsZXRvbml6ZScpCmNvbmZ1c2lvbl9tYXRyaXgoaW1hZ2VfbGV2ZWwsICdmaWx0ZXJfMjQwX3NrZWxldG9uaXplZCcpCmBgYAoKCgojIyMgU2VnbWVudGF0aW9uczogemVybyBhbGwgcGl4ZWxzIDwgMjUwCgpgYGB7cn0KY29tcHV0ZV9hdXByKGltYWdlX2xldmVsJGZpbHRlcl8yNTAsIGltYWdlX2xldmVsJGxhYmVsLCAnUFI6IFNlZ21lbnRhdGlvbnM6IHplcm8gYWxsIHBpeGVscyA8IDI1MCcpCmNvbXB1dGVfYXVyb2MoaW1hZ2VfbGV2ZWwkZmlsdGVyXzI1MCwgaW1hZ2VfbGV2ZWwkbGFiZWwsICdST0M6IFNlZ21lbnRhdGlvbnM6IHplcm8gYWxsIHBpeGVscyA8IDI1MCcpCmNvbmZ1c2lvbl9tYXRyaXgoaW1hZ2VfbGV2ZWwsICdmaWx0ZXJfMjUwJykKYGBgCgpgYGB7cn0KY29tcHV0ZV9hdXByKGltYWdlX2xldmVsJGZpbHRlcl8yNTBfYmluYXJpemVkLCBpbWFnZV9sZXZlbCRsYWJlbCwgJ1BSOiBTZWdtZW50YXRpb25zOiB6ZXJvIGFsbCBwaXhlbHMgPCAyNTAgYW5kIGJpbmFyaXplJykKY29tcHV0ZV9hdXJvYyhpbWFnZV9sZXZlbCRmaWx0ZXJfMjUwX2JpbmFyaXplZCwgaW1hZ2VfbGV2ZWwkbGFiZWwsICdST0M6IFNlZ21lbnRhdGlvbnM6IHplcm8gYWxsIHBpeGVscyA8IDI1MCBhbmQgYmluYXJpemUnKQpjb25mdXNpb25fbWF0cml4KGltYWdlX2xldmVsLCAnZmlsdGVyXzI1MF9iaW5hcml6ZWQnKQpgYGAKCmBgYHtyfQpjb21wdXRlX2F1cHIoaW1hZ2VfbGV2ZWwkZmlsdGVyXzI1MF9za2VsZXRvbml6ZWQsIGltYWdlX2xldmVsJGxhYmVsLCAnUFI6IFNlZ21lbnRhdGlvbnM6IHplcm8gYWxsIHBpeGVscyA8IDI1MCBhbmQgc2tlbGV0b25pemUnKQpjb21wdXRlX2F1cm9jKGltYWdlX2xldmVsJGZpbHRlcl8yNTBfc2tlbGV0b25pemVkLCBpbWFnZV9sZXZlbCRsYWJlbCwgJ1JPQzogU2VnbWVudGF0aW9uczogemVybyBhbGwgcGl4ZWxzIDwgMjUwIGFuZCBza2VsZXRvbml6ZScpCmNvbmZ1c2lvbl9tYXRyaXgoaW1hZ2VfbGV2ZWwsICdmaWx0ZXJfMjUwX3NrZWxldG9uaXplZCcpCmBgYAoKCgojIyMgU2VnbWVudGF0aW9uczogemVybyBhbGwgcGl4ZWxzIDwgMjU3CgpgYGB7cn0KY29tcHV0ZV9hdXByKGltYWdlX2xldmVsJGZpbHRlcl8yNTcsIGltYWdlX2xldmVsJGxhYmVsLCAnUFI6IFNlZ21lbnRhdGlvbnM6IHplcm8gYWxsIHBpeGVscyA8IDI1NycpCmNvbXB1dGVfYXVyb2MoaW1hZ2VfbGV2ZWwkZmlsdGVyXzI1NywgaW1hZ2VfbGV2ZWwkbGFiZWwsICdST0M6IFNlZ21lbnRhdGlvbnM6IHplcm8gYWxsIHBpeGVscyA8IDI1NycpCmNvbmZ1c2lvbl9tYXRyaXgoaW1hZ2VfbGV2ZWwsICdmaWx0ZXJfMjU3JykKYGBgCgpgYGB7cn0KY29tcHV0ZV9hdXByKGltYWdlX2xldmVsJGZpbHRlcl8yNTdfYmluYXJpemVkLCBpbWFnZV9sZXZlbCRsYWJlbCwgJ1BSOiBTZWdtZW50YXRpb25zOiB6ZXJvIGFsbCBwaXhlbHMgPCAyNTcgYW5kIGJpbmFyaXplJykKY29tcHV0ZV9hdXJvYyhpbWFnZV9sZXZlbCRmaWx0ZXJfMjU3X2JpbmFyaXplZCwgaW1hZ2VfbGV2ZWwkbGFiZWwsICdST0M6IFNlZ21lbnRhdGlvbnM6IHplcm8gYWxsIHBpeGVscyA8IDI1NyBhbmQgYmluYXJpemUnKQpjb25mdXNpb25fbWF0cml4KGltYWdlX2xldmVsLCAnZmlsdGVyXzI1N19iaW5hcml6ZWQnKQpgYGAKCmBgYHtyfQpjb21wdXRlX2F1cHIoaW1hZ2VfbGV2ZWwkZmlsdGVyXzI1N19za2VsZXRvbml6ZWQsIGltYWdlX2xldmVsJGxhYmVsLCAnUFI6IFNlZ21lbnRhdGlvbnM6IHplcm8gYWxsIHBpeGVscyA8IDI1NyBhbmQgc2tlbGV0b25pemUnKQpjb21wdXRlX2F1cm9jKGltYWdlX2xldmVsJGZpbHRlcl8yNTdfc2tlbGV0b25pemVkLCBpbWFnZV9sZXZlbCRsYWJlbCwgJ1JPQzogU2VnbWVudGF0aW9uczogemVybyBhbGwgcGl4ZWxzIDwgMjU3IGFuZCBza2VsZXRvbml6ZScpCmNvbmZ1c2lvbl9tYXRyaXgoaW1hZ2VfbGV2ZWwsICdmaWx0ZXJfMjU3X3NrZWxldG9uaXplZCcpCmBgYAoKCgojIyMgU2VnbWVudGF0aW9uczogemVybyBhbGwgcGl4ZWxzID4gMTAKCmBgYHtyfQpjb21wdXRlX2F1cHIoaW1hZ2VfbGV2ZWwkZmlsdGVyXzEwLCBpbWFnZV9sZXZlbCRsYWJlbCwgJ1BSOiBTZWdtZW50YXRpb25zOiB6ZXJvIGFsbCBwaXhlbHMgPCAxMCcpCmNvbXB1dGVfYXVyb2MoaW1hZ2VfbGV2ZWwkZmlsdGVyXzEwLCBpbWFnZV9sZXZlbCRsYWJlbCwgJ1JPQzogU2VnbWVudGF0aW9uczogemVybyBhbGwgcGl4ZWxzIDwgMTAnKQpjb25mdXNpb25fbWF0cml4KGltYWdlX2xldmVsLCAnZmlsdGVyXzEwJykKYGBgCgpgYGB7cn0KY29tcHV0ZV9hdXByKGltYWdlX2xldmVsJGZpbHRlcl8xMF9iaW5hcml6ZWQsIGltYWdlX2xldmVsJGxhYmVsLCAnUFI6IFNlZ21lbnRhdGlvbnM6IHplcm8gYWxsIHBpeGVscyA8IDEwIGFuZCBiaW5hcml6ZScpCmNvbXB1dGVfYXVyb2MoaW1hZ2VfbGV2ZWwkZmlsdGVyXzEwX2JpbmFyaXplZCwgaW1hZ2VfbGV2ZWwkbGFiZWwsICdST0M6IFNlZ21lbnRhdGlvbnM6IHplcm8gYWxsIHBpeGVscyA8IDEwIGFuZCBiaW5hcml6ZScpCmNvbmZ1c2lvbl9tYXRyaXgoaW1hZ2VfbGV2ZWwsICdmaWx0ZXJfMTBfYmluYXJpemVkJykKYGBgCgpgYGB7cn0KY29tcHV0ZV9hdXByKGltYWdlX2xldmVsJGZpbHRlcl8xMF9za2VsZXRvbml6ZWQsIGltYWdlX2xldmVsJGxhYmVsLCAnUFI6IFNlZ21lbnRhdGlvbnM6IHplcm8gYWxsIHBpeGVscyA8IDEwIGFuZCBza2VsZXRvbml6ZScpCmNvbXB1dGVfYXVyb2MoaW1hZ2VfbGV2ZWwkZmlsdGVyXzEwX3NrZWxldG9uaXplZCwgaW1hZ2VfbGV2ZWwkbGFiZWwsICdST0M6IFNlZ21lbnRhdGlvbnM6IHplcm8gYWxsIHBpeGVscyA8IDEwIGFuZCBza2VsZXRvbml6ZScpCmNvbmZ1c2lvbl9tYXRyaXgoaW1hZ2VfbGV2ZWwsICdmaWx0ZXJfMTBfc2tlbGV0b25pemVkJykKYGBgCgoKCiMjIyBTZWdtZW50YXRpb25zOiB6ZXJvIGFsbCBwaXhlbHMgPCA3NSBhbmQgPiAxNTAKCmBgYHtyfQpjb21wdXRlX2F1cHIoaW1hZ2VfbGV2ZWwkZmlsdGVyXzc1LCBpbWFnZV9sZXZlbCRsYWJlbCwgJ1BSOiBTZWdtZW50YXRpb25zOiB6ZXJvIGFsbCBwaXhlbHMgPCA3NSBhbmQgPiAxNTAnKQpjb21wdXRlX2F1cm9jKGltYWdlX2xldmVsJGZpbHRlcl83NSwgaW1hZ2VfbGV2ZWwkbGFiZWwsICdST0M6IFNlZ21lbnRhdGlvbnM6IHplcm8gYWxsIHBpeGVscyA8IDc1IGFuZCA+IDE1MCcpCmNvbmZ1c2lvbl9tYXRyaXgoaW1hZ2VfbGV2ZWwsICdmaWx0ZXJfNzUnKQpgYGAKCmBgYHtyfQpjb21wdXRlX2F1cHIoaW1hZ2VfbGV2ZWwkZmlsdGVyXzc1X2JpbmFyaXplZCwgaW1hZ2VfbGV2ZWwkbGFiZWwsICdQUjogU2VnbWVudGF0aW9uczogemVybyBhbGwgcGl4ZWxzIDwgNzUgYW5kID4gMTUwIGFuZCBiaW5hcml6ZScpCmNvbXB1dGVfYXVyb2MoaW1hZ2VfbGV2ZWwkZmlsdGVyXzc1X2JpbmFyaXplZCwgaW1hZ2VfbGV2ZWwkbGFiZWwsICdST0M6IFNlZ21lbnRhdGlvbnM6IHplcm8gYWxsIHBpeGVscyA8IDc1IGFuZCA+IDE1MCBhbmQgYmluYXJpemUnKQpjb25mdXNpb25fbWF0cml4KGltYWdlX2xldmVsLCAnZmlsdGVyXzc1X2JpbmFyaXplZCcpCmBgYAoKYGBge3J9CmNvbXB1dGVfYXVwcihpbWFnZV9sZXZlbCRmaWx0ZXJfNzVfc2tlbGV0b25pemVkLCBpbWFnZV9sZXZlbCRsYWJlbCwgJ1BSOiBTZWdtZW50YXRpb25zOiB6ZXJvIGFsbCBwaXhlbHMgPCA3NSBhbmQgPiAxNTAgYW5kIHNrZWxldG9uaXplJykKY29tcHV0ZV9hdXJvYyhpbWFnZV9sZXZlbCRmaWx0ZXJfNzVfc2tlbGV0b25pemVkLCBpbWFnZV9sZXZlbCRsYWJlbCwgJ1JPQzogU2VnbWVudGF0aW9uczogemVybyBhbGwgcGl4ZWxzIDwgNzUgYW5kID4gMTUwIGFuZCBza2VsZXRvbml6ZScpCmNvbmZ1c2lvbl9tYXRyaXgoaW1hZ2VfbGV2ZWwsICdmaWx0ZXJfNzVfc2tlbGV0b25pemVkJykKYGBgCgoKCgoKCgoKCgoKIyBTdWJqZWN0LWxldmVsIEFuYWx5c2lzCgojIyMgUmV0Q2FtIEltYWdlcwoKYGBge3J9CmNvbXB1dGVfYXVwcihzdWJqZWN0X2xldmVsJHJldGNhbSwgc3ViamVjdF9sZXZlbCRsYWJlbCwgJ1BSOiBSYXcgUmV0Q2FtIEltYWdlcycpCmNvbXB1dGVfYXVyb2Moc3ViamVjdF9sZXZlbCRyZXRjYW0sIHN1YmplY3RfbGV2ZWwkbGFiZWwsICdST0M6IFJhdyBSZXRDYW0gSW1hZ2VzJykKY29uZnVzaW9uX21hdHJpeChzdWJqZWN0X2xldmVsLCAncmV0Y2FtJykKYGBgCgpgYGB7cn0KY29tcHV0ZV9hdXByKHN1YmplY3RfbGV2ZWwkcmV0Y2FtX3JhbmRvbSwgc3ViamVjdF9sZXZlbCRsYWJlbCwgJ1BSOiBSYXcgUmV0Q2FtIEltYWdlcyAtIFNodWZmbGVkIExhYmVscycpCmNvbXB1dGVfYXVyb2Moc3ViamVjdF9sZXZlbCRyZXRjYW1fcmFuZG9tLCBzdWJqZWN0X2xldmVsJGxhYmVsLCAnUk9DOiBSYXcgUmV0Q2FtIEltYWdlcyAtIFNodWZmbGVkIExhYmVscycpCmNvbmZ1c2lvbl9tYXRyaXgoc3ViamVjdF9sZXZlbCwgJ3JldGNhbV9yYW5kb20nKQpgYGAKCgoKIyMjIFNlZ21lbnRhdGlvbnM6IHplcm8gYWxsIHBpeGVscyA8IDAKCmBgYHtyfQpjb21wdXRlX2F1cHIoc3ViamVjdF9sZXZlbCRmaWx0ZXJfMCwgc3ViamVjdF9sZXZlbCRsYWJlbCwgJ1BSOiBTZWdtZW50YXRpb25zOiB6ZXJvIGFsbCBwaXhlbHMgPCAwJykKY29tcHV0ZV9hdXJvYyhzdWJqZWN0X2xldmVsJGZpbHRlcl8wLCBzdWJqZWN0X2xldmVsJGxhYmVsLCAnUk9DOiBTZWdtZW50YXRpb25zOiB6ZXJvIGFsbCBwaXhlbHMgPCAwJykKY29uZnVzaW9uX21hdHJpeChzdWJqZWN0X2xldmVsLCAnZmlsdGVyXzAnKQpgYGAKCmBgYHtyfQpjb21wdXRlX2F1cHIoc3ViamVjdF9sZXZlbCRmaWx0ZXJfMF9iaW5hcml6ZWQsIHN1YmplY3RfbGV2ZWwkbGFiZWwsICdQUjogU2VnbWVudGF0aW9uczogemVybyBhbGwgcGl4ZWxzIDwgMCBhbmQgYmluYXJpemUnKQpjb21wdXRlX2F1cm9jKHN1YmplY3RfbGV2ZWwkZmlsdGVyXzBfYmluYXJpemVkLCBzdWJqZWN0X2xldmVsJGxhYmVsLCAnUk9DOiBTZWdtZW50YXRpb25zOiB6ZXJvIGFsbCBwaXhlbHMgPCAwIGFuZCBiaW5hcml6ZScpCmNvbmZ1c2lvbl9tYXRyaXgoc3ViamVjdF9sZXZlbCwgJ2ZpbHRlcl8wX2JpbmFyaXplZCcpCmBgYAoKYGBge3J9CmNvbXB1dGVfYXVwcihzdWJqZWN0X2xldmVsJGZpbHRlcl8wX3NrZWxldG9uaXplZCwgc3ViamVjdF9sZXZlbCRsYWJlbCwgJ1BSOiBTZWdtZW50YXRpb25zOiB6ZXJvIGFsbCBwaXhlbHMgPCAwIGFuZCBza2VsZXRvbml6ZScpCmNvbXB1dGVfYXVyb2Moc3ViamVjdF9sZXZlbCRmaWx0ZXJfMF9za2VsZXRvbml6ZWQsIHN1YmplY3RfbGV2ZWwkbGFiZWwsICdST0M6IFNlZ21lbnRhdGlvbnM6IHplcm8gYWxsIHBpeGVscyA8IDAgYW5kIHNrZWxldG9uaXplJykKY29uZnVzaW9uX21hdHJpeChzdWJqZWN0X2xldmVsLCAnZmlsdGVyXzBfc2tlbGV0b25pemVkJykKYGBgCgpgYGB7cn0KY29tcHV0ZV9hdXByKHN1YmplY3RfbGV2ZWwkZmlsdGVyXzBfcmFuZG9tLCBzdWJqZWN0X2xldmVsJGxhYmVsLCAnUFI6IFNlZ21lbnRhdGlvbnM6IHplcm8gYWxsIHBpeGVscyA8IDAgLSBTaHVmZmxlZCBMYWJlbHMnKQpjb21wdXRlX2F1cm9jKHN1YmplY3RfbGV2ZWwkZmlsdGVyXzBfcmFuZG9tLCBzdWJqZWN0X2xldmVsJGxhYmVsLCAnUk9DOiBTZWdtZW50YXRpb25zOiB6ZXJvIGFsbCBwaXhlbHMgPCAwIC0gU2h1ZmZsZWQgTGFiZWxzJykKY29uZnVzaW9uX21hdHJpeChzdWJqZWN0X2xldmVsLCAnZmlsdGVyXzBfcmFuZG9tJykKYGBgCgoKIyMjIFNlZ21lbnRhdGlvbnM6IHplcm8gYWxsIHBpeGVscyA8IDUwCgpgYGB7cn0KY29tcHV0ZV9hdXByKHN1YmplY3RfbGV2ZWwkZmlsdGVyXzUwLCBzdWJqZWN0X2xldmVsJGxhYmVsLCAnUFI6IFNlZ21lbnRhdGlvbnM6IHplcm8gYWxsIHBpeGVscyA8IDUwJykKY29tcHV0ZV9hdXJvYyhzdWJqZWN0X2xldmVsJGZpbHRlcl81MCwgc3ViamVjdF9sZXZlbCRsYWJlbCwgJ1JPQzogU2VnbWVudGF0aW9uczogemVybyBhbGwgcGl4ZWxzIDwgNTAnKQpjb25mdXNpb25fbWF0cml4KHN1YmplY3RfbGV2ZWwsICdmaWx0ZXJfNTAnKQpgYGAKCmBgYHtyfQpjb21wdXRlX2F1cHIoc3ViamVjdF9sZXZlbCRmaWx0ZXJfNTBfYmluYXJpemVkLCBzdWJqZWN0X2xldmVsJGxhYmVsLCAnUFI6IFNlZ21lbnRhdGlvbnM6IHplcm8gYWxsIHBpeGVscyA8IDUwIGFuZCBiaW5hcml6ZScpCmNvbXB1dGVfYXVyb2Moc3ViamVjdF9sZXZlbCRmaWx0ZXJfNTBfYmluYXJpemVkLCBzdWJqZWN0X2xldmVsJGxhYmVsLCAnUk9DOiBTZWdtZW50YXRpb25zOiB6ZXJvIGFsbCBwaXhlbHMgPCA1MCBhbmQgYmluYXJpemUnKQpjb25mdXNpb25fbWF0cml4KHN1YmplY3RfbGV2ZWwsICdmaWx0ZXJfMF9iaW5hcml6ZWQnKQpgYGAKCmBgYHtyfQpjb21wdXRlX2F1cHIoc3ViamVjdF9sZXZlbCRmaWx0ZXJfNTBfc2tlbGV0b25pemVkLCBzdWJqZWN0X2xldmVsJGxhYmVsLCAnUFI6IFNlZ21lbnRhdGlvbnM6IHplcm8gYWxsIHBpeGVscyA8IDUwIGFuZCBza2VsZXRvbml6ZScpCmNvbXB1dGVfYXVyb2Moc3ViamVjdF9sZXZlbCRmaWx0ZXJfNTBfc2tlbGV0b25pemVkLCBzdWJqZWN0X2xldmVsJGxhYmVsLCAnUk9DOiBTZWdtZW50YXRpb25zOiB6ZXJvIGFsbCBwaXhlbHMgPCA1MCBhbmQgc2tlbGV0b25pemUnKQpjb25mdXNpb25fbWF0cml4KHN1YmplY3RfbGV2ZWwsICdmaWx0ZXJfNTBfc2tlbGV0b25pemVkJykKYGBgCgoKCiMjIyBTZWdtZW50YXRpb25zOiB6ZXJvIGFsbCBwaXhlbHMgPCAxMDAKCmBgYHtyfQpjb21wdXRlX2F1cHIoc3ViamVjdF9sZXZlbCRmaWx0ZXJfMTAwLCBzdWJqZWN0X2xldmVsJGxhYmVsLCAnUFI6IFNlZ21lbnRhdGlvbnM6IHplcm8gYWxsIHBpeGVscyA8IDEwMCcpCmNvbXB1dGVfYXVyb2Moc3ViamVjdF9sZXZlbCRmaWx0ZXJfMTAwLCBzdWJqZWN0X2xldmVsJGxhYmVsLCAnUk9DOiBTZWdtZW50YXRpb25zOiB6ZXJvIGFsbCBwaXhlbHMgPCAxMDAnKQpjb25mdXNpb25fbWF0cml4KHN1YmplY3RfbGV2ZWwsICdmaWx0ZXJfMTAwJykKYGBgCgpgYGB7cn0KY29tcHV0ZV9hdXByKHN1YmplY3RfbGV2ZWwkZmlsdGVyXzEwMF9iaW5hcml6ZWQsIHN1YmplY3RfbGV2ZWwkbGFiZWwsICdQUjogU2VnbWVudGF0aW9uczogemVybyBhbGwgcGl4ZWxzIDwgMTAwIGFuZCBiaW5hcml6ZScpCmNvbXB1dGVfYXVyb2Moc3ViamVjdF9sZXZlbCRmaWx0ZXJfMTAwX2JpbmFyaXplZCwgc3ViamVjdF9sZXZlbCRsYWJlbCwgJ1JPQzogU2VnbWVudGF0aW9uczogemVybyBhbGwgcGl4ZWxzIDwgMTAwIGFuZCBiaW5hcml6ZScpCmNvbmZ1c2lvbl9tYXRyaXgoc3ViamVjdF9sZXZlbCwgJ2ZpbHRlcl8xMDBfYmluYXJpemVkJykKYGBgCgpgYGB7cn0KY29tcHV0ZV9hdXByKHN1YmplY3RfbGV2ZWwkZmlsdGVyXzEwMF9za2VsZXRvbml6ZWQsIHN1YmplY3RfbGV2ZWwkbGFiZWwsICdQUjogU2VnbWVudGF0aW9uczogemVybyBhbGwgcGl4ZWxzIDwgMTAwIGFuZCBza2VsZXRvbml6ZScpCmNvbXB1dGVfYXVyb2Moc3ViamVjdF9sZXZlbCRmaWx0ZXJfMTAwX3NrZWxldG9uaXplZCwgc3ViamVjdF9sZXZlbCRsYWJlbCwgJ1JPQzogU2VnbWVudGF0aW9uczogemVybyBhbGwgcGl4ZWxzIDwgMTAwIGFuZCBza2VsZXRvbml6ZScpCmNvbmZ1c2lvbl9tYXRyaXgoc3ViamVjdF9sZXZlbCwgJ2ZpbHRlcl8xMDBfc2tlbGV0b25pemVkJykKYGBgCgoKCiMjIyBTZWdtZW50YXRpb25zOiB6ZXJvIGFsbCBwaXhlbHMgPCAxNTAKCmBgYHtyfQpjb21wdXRlX2F1cHIoc3ViamVjdF9sZXZlbCRmaWx0ZXJfMTUwLCBzdWJqZWN0X2xldmVsJGxhYmVsLCAnUFI6IFNlZ21lbnRhdGlvbnM6IHplcm8gYWxsIHBpeGVscyA8IDE1MCcpCmNvbXB1dGVfYXVyb2Moc3ViamVjdF9sZXZlbCRmaWx0ZXJfMTUwLCBzdWJqZWN0X2xldmVsJGxhYmVsLCAnUk9DOiBTZWdtZW50YXRpb25zOiB6ZXJvIGFsbCBwaXhlbHMgPCAxNTAnKQpjb25mdXNpb25fbWF0cml4KHN1YmplY3RfbGV2ZWwsICdmaWx0ZXJfMTUwJykKYGBgCgpgYGB7cn0KY29tcHV0ZV9hdXByKHN1YmplY3RfbGV2ZWwkZmlsdGVyXzE1MF9iaW5hcml6ZWQsIHN1YmplY3RfbGV2ZWwkbGFiZWwsICdQUjogU2VnbWVudGF0aW9uczogemVybyBhbGwgcGl4ZWxzIDwgMTUwIGFuZCBiaW5hcml6ZScpCmNvbXB1dGVfYXVyb2Moc3ViamVjdF9sZXZlbCRmaWx0ZXJfMTUwX2JpbmFyaXplZCwgc3ViamVjdF9sZXZlbCRsYWJlbCwgJ1JPQzogU2VnbWVudGF0aW9uczogemVybyBhbGwgcGl4ZWxzIDwgMTUwIGFuZCBiaW5hcml6ZScpCmNvbmZ1c2lvbl9tYXRyaXgoc3ViamVjdF9sZXZlbCwgJ2ZpbHRlcl8xNTBfYmluYXJpemVkJykKYGBgCgpgYGB7cn0KY29tcHV0ZV9hdXByKHN1YmplY3RfbGV2ZWwkZmlsdGVyXzE1MF9za2VsZXRvbml6ZWQsIHN1YmplY3RfbGV2ZWwkbGFiZWwsICdQUjogU2VnbWVudGF0aW9uczogemVybyBhbGwgcGl4ZWxzIDwgMTUwIGFuZCBza2VsZXRvbml6ZScpCmNvbXB1dGVfYXVyb2Moc3ViamVjdF9sZXZlbCRmaWx0ZXJfMTUwX3NrZWxldG9uaXplZCwgc3ViamVjdF9sZXZlbCRsYWJlbCwgJ1JPQzogU2VnbWVudGF0aW9uczogemVybyBhbGwgcGl4ZWxzIDwgMTUwIGFuZCBza2VsZXRvbml6ZScpCmNvbmZ1c2lvbl9tYXRyaXgoc3ViamVjdF9sZXZlbCwgJ2ZpbHRlcl8xNTBfc2tlbGV0b25pemVkJykKYGBgCgoKCiMjIyBTZWdtZW50YXRpb25zOiB6ZXJvIGFsbCBwaXhlbHMgPCAyMDAKCmBgYHtyfQpjb21wdXRlX2F1cHIoc3ViamVjdF9sZXZlbCRmaWx0ZXJfMjAwLCBzdWJqZWN0X2xldmVsJGxhYmVsLCAnUFI6IFNlZ21lbnRhdGlvbnM6IHplcm8gYWxsIHBpeGVscyA8IDIwMCcpCmNvbXB1dGVfYXVyb2Moc3ViamVjdF9sZXZlbCRmaWx0ZXJfMjAwLCBzdWJqZWN0X2xldmVsJGxhYmVsLCAnUk9DOiBTZWdtZW50YXRpb25zOiB6ZXJvIGFsbCBwaXhlbHMgPCAyMDAnKQpjb25mdXNpb25fbWF0cml4KHN1YmplY3RfbGV2ZWwsICdmaWx0ZXJfMjAwJykKYGBgCgpgYGB7cn0KY29tcHV0ZV9hdXByKHN1YmplY3RfbGV2ZWwkZmlsdGVyXzIwMF9iaW5hcml6ZWQsIHN1YmplY3RfbGV2ZWwkbGFiZWwsICdQUjogU2VnbWVudGF0aW9uczogemVybyBhbGwgcGl4ZWxzIDwgMjAwIGFuZCBiaW5hcml6ZScpCmNvbXB1dGVfYXVyb2Moc3ViamVjdF9sZXZlbCRmaWx0ZXJfMjAwX2JpbmFyaXplZCwgc3ViamVjdF9sZXZlbCRsYWJlbCwgJ1JPQzogU2VnbWVudGF0aW9uczogemVybyBhbGwgcGl4ZWxzIDwgMjAwIGFuZCBiaW5hcml6ZScpCmNvbmZ1c2lvbl9tYXRyaXgoc3ViamVjdF9sZXZlbCwgJ2ZpbHRlcl8yMDBfYmluYXJpemVkJykKYGBgCgpgYGB7cn0KY29tcHV0ZV9hdXByKHN1YmplY3RfbGV2ZWwkZmlsdGVyXzIwMF9za2VsZXRvbml6ZWQsIHN1YmplY3RfbGV2ZWwkbGFiZWwsICdQUjogU2VnbWVudGF0aW9uczogemVybyBhbGwgcGl4ZWxzIDwgMjAwIGFuZCBza2VsZXRvbml6ZScpCmNvbXB1dGVfYXVyb2Moc3ViamVjdF9sZXZlbCRmaWx0ZXJfMjAwX3NrZWxldG9uaXplZCwgc3ViamVjdF9sZXZlbCRsYWJlbCwgJ1JPQzogU2VnbWVudGF0aW9uczogemVybyBhbGwgcGl4ZWxzIDwgMjAwIGFuZCBza2VsZXRvbml6ZScpCmNvbmZ1c2lvbl9tYXRyaXgoc3ViamVjdF9sZXZlbCwgJ2ZpbHRlcl8yMDBfc2tlbGV0b25pemVkJykKYGBgCgoKCiMjIyBTZWdtZW50YXRpb25zOiB6ZXJvIGFsbCBwaXhlbHMgPCAyMTAKCmBgYHtyfQpjb21wdXRlX2F1cHIoc3ViamVjdF9sZXZlbCRmaWx0ZXJfMjEwLCBzdWJqZWN0X2xldmVsJGxhYmVsLCAnUFI6IFNlZ21lbnRhdGlvbnM6IHplcm8gYWxsIHBpeGVscyA8IDIxMCcpCmNvbXB1dGVfYXVyb2Moc3ViamVjdF9sZXZlbCRmaWx0ZXJfMjEwLCBzdWJqZWN0X2xldmVsJGxhYmVsLCAnUk9DOiBTZWdtZW50YXRpb25zOiB6ZXJvIGFsbCBwaXhlbHMgPCAyMTAnKQpjb25mdXNpb25fbWF0cml4KHN1YmplY3RfbGV2ZWwsICdmaWx0ZXJfMjEwJykKYGBgCgpgYGB7cn0KY29tcHV0ZV9hdXByKHN1YmplY3RfbGV2ZWwkZmlsdGVyXzIxMF9iaW5hcml6ZWQsIHN1YmplY3RfbGV2ZWwkbGFiZWwsICdQUjogU2VnbWVudGF0aW9uczogemVybyBhbGwgcGl4ZWxzIDwgMjEwIGFuZCBiaW5hcml6ZScpCmNvbXB1dGVfYXVyb2Moc3ViamVjdF9sZXZlbCRmaWx0ZXJfMjEwX2JpbmFyaXplZCwgc3ViamVjdF9sZXZlbCRsYWJlbCwgJ1JPQzogU2VnbWVudGF0aW9uczogemVybyBhbGwgcGl4ZWxzIDwgMjEwIGFuZCBiaW5hcml6ZScpCmNvbmZ1c2lvbl9tYXRyaXgoc3ViamVjdF9sZXZlbCwgJ2ZpbHRlcl8yMTBfYmluYXJpemVkJykKYGBgCgpgYGB7cn0KY29tcHV0ZV9hdXByKHN1YmplY3RfbGV2ZWwkZmlsdGVyXzIxMF9za2VsZXRvbml6ZWQsIHN1YmplY3RfbGV2ZWwkbGFiZWwsICdQUjogU2VnbWVudGF0aW9uczogemVybyBhbGwgcGl4ZWxzIDwgMjEwIGFuZCBza2VsZXRvbml6ZScpCmNvbXB1dGVfYXVyb2Moc3ViamVjdF9sZXZlbCRmaWx0ZXJfMjEwX3NrZWxldG9uaXplZCwgc3ViamVjdF9sZXZlbCRsYWJlbCwgJ1JPQzogU2VnbWVudGF0aW9uczogemVybyBhbGwgcGl4ZWxzIDwgMjEwIGFuZCBza2VsZXRvbml6ZScpCmNvbmZ1c2lvbl9tYXRyaXgoc3ViamVjdF9sZXZlbCwgJ2ZpbHRlcl8yMTBfc2tlbGV0b25pemVkJykKYGBgCgoKCiMjIyBTZWdtZW50YXRpb25zOiB6ZXJvIGFsbCBwaXhlbHMgPCAyMjAKCmBgYHtyfQpjb21wdXRlX2F1cHIoc3ViamVjdF9sZXZlbCRmaWx0ZXJfMjIwLCBzdWJqZWN0X2xldmVsJGxhYmVsLCAnUFI6IFNlZ21lbnRhdGlvbnM6IHplcm8gYWxsIHBpeGVscyA8IDIyMCcpCmNvbXB1dGVfYXVyb2Moc3ViamVjdF9sZXZlbCRmaWx0ZXJfMjIwLCBzdWJqZWN0X2xldmVsJGxhYmVsLCAnUk9DOiBTZWdtZW50YXRpb25zOiB6ZXJvIGFsbCBwaXhlbHMgPCAyMjAnKQpjb25mdXNpb25fbWF0cml4KHN1YmplY3RfbGV2ZWwsICdmaWx0ZXJfMjIwJykKYGBgCgpgYGB7cn0KY29tcHV0ZV9hdXByKHN1YmplY3RfbGV2ZWwkZmlsdGVyXzIyMF9iaW5hcml6ZWQsIHN1YmplY3RfbGV2ZWwkbGFiZWwsICdQUjogU2VnbWVudGF0aW9uczogemVybyBhbGwgcGl4ZWxzIDwgMjIwIGFuZCBiaW5hcml6ZScpCmNvbXB1dGVfYXVyb2Moc3ViamVjdF9sZXZlbCRmaWx0ZXJfMjIwX2JpbmFyaXplZCwgc3ViamVjdF9sZXZlbCRsYWJlbCwgJ1JPQzogU2VnbWVudGF0aW9uczogemVybyBhbGwgcGl4ZWxzIDwgMjIwIGFuZCBiaW5hcml6ZScpCmNvbmZ1c2lvbl9tYXRyaXgoc3ViamVjdF9sZXZlbCwgJ2ZpbHRlcl8yMjBfYmluYXJpemVkJykKYGBgCgpgYGB7cn0KY29tcHV0ZV9hdXByKHN1YmplY3RfbGV2ZWwkZmlsdGVyXzIyMF9za2VsZXRvbml6ZWQsIHN1YmplY3RfbGV2ZWwkbGFiZWwsICdQUjogU2VnbWVudGF0aW9uczogemVybyBhbGwgcGl4ZWxzIDwgMjIwIGFuZCBza2VsZXRvbml6ZScpCmNvbXB1dGVfYXVyb2Moc3ViamVjdF9sZXZlbCRmaWx0ZXJfMjIwX3NrZWxldG9uaXplZCwgc3ViamVjdF9sZXZlbCRsYWJlbCwgJ1JPQzogU2VnbWVudGF0aW9uczogemVybyBhbGwgcGl4ZWxzIDwgMjIwIGFuZCBza2VsZXRvbml6ZScpCmNvbmZ1c2lvbl9tYXRyaXgoc3ViamVjdF9sZXZlbCwgJ2ZpbHRlcl8yMjBfc2tlbGV0b25pemVkJykKYGBgCgoKCiMjIyBTZWdtZW50YXRpb25zOiB6ZXJvIGFsbCBwaXhlbHMgPCAyMzAKCmBgYHtyfQpjb21wdXRlX2F1cHIoc3ViamVjdF9sZXZlbCRmaWx0ZXJfMjMwLCBzdWJqZWN0X2xldmVsJGxhYmVsLCAnUFI6IFNlZ21lbnRhdGlvbnM6IHplcm8gYWxsIHBpeGVscyA8IDIzMCcpCmNvbXB1dGVfYXVyb2Moc3ViamVjdF9sZXZlbCRmaWx0ZXJfMjMwLCBzdWJqZWN0X2xldmVsJGxhYmVsLCAnUk9DOiBTZWdtZW50YXRpb25zOiB6ZXJvIGFsbCBwaXhlbHMgPCAyMzAnKQpjb25mdXNpb25fbWF0cml4KHN1YmplY3RfbGV2ZWwsICdmaWx0ZXJfMjMwJykKYGBgCgpgYGB7cn0KY29tcHV0ZV9hdXByKHN1YmplY3RfbGV2ZWwkZmlsdGVyXzIzMF9iaW5hcml6ZWQsIHN1YmplY3RfbGV2ZWwkbGFiZWwsICdQUjogU2VnbWVudGF0aW9uczogemVybyBhbGwgcGl4ZWxzIDwgMjMwIGFuZCBiaW5hcml6ZScpCmNvbXB1dGVfYXVyb2Moc3ViamVjdF9sZXZlbCRmaWx0ZXJfMjMwX2JpbmFyaXplZCwgc3ViamVjdF9sZXZlbCRsYWJlbCwgJ1JPQzogU2VnbWVudGF0aW9uczogemVybyBhbGwgcGl4ZWxzIDwgMjMwIGFuZCBiaW5hcml6ZScpCmNvbmZ1c2lvbl9tYXRyaXgoc3ViamVjdF9sZXZlbCwgJ2ZpbHRlcl8yMzBfYmluYXJpemVkJykKYGBgCgpgYGB7cn0KY29tcHV0ZV9hdXByKHN1YmplY3RfbGV2ZWwkZmlsdGVyXzIzMF9za2VsZXRvbml6ZWQsIHN1YmplY3RfbGV2ZWwkbGFiZWwsICdQUjogU2VnbWVudGF0aW9uczogemVybyBhbGwgcGl4ZWxzIDwgMjMwIGFuZCBza2VsZXRvbml6ZScpCmNvbXB1dGVfYXVyb2Moc3ViamVjdF9sZXZlbCRmaWx0ZXJfMjMwX3NrZWxldG9uaXplZCwgc3ViamVjdF9sZXZlbCRsYWJlbCwgJ1JPQzogU2VnbWVudGF0aW9uczogemVybyBhbGwgcGl4ZWxzIDwgMjMwIGFuZCBza2VsZXRvbml6ZScpCmNvbmZ1c2lvbl9tYXRyaXgoc3ViamVjdF9sZXZlbCwgJ2ZpbHRlcl8yMzBfc2tlbGV0b25pemVkJykKYGBgCgoKCiMjIyBTZWdtZW50YXRpb25zOiB6ZXJvIGFsbCBwaXhlbHMgPCAyNDAKCmBgYHtyfQpjb21wdXRlX2F1cHIoc3ViamVjdF9sZXZlbCRmaWx0ZXJfMjQwLCBzdWJqZWN0X2xldmVsJGxhYmVsLCAnUFI6IFNlZ21lbnRhdGlvbnM6IHplcm8gYWxsIHBpeGVscyA8IDI0MCcpCmNvbXB1dGVfYXVyb2Moc3ViamVjdF9sZXZlbCRmaWx0ZXJfMjQwLCBzdWJqZWN0X2xldmVsJGxhYmVsLCAnUk9DOiBTZWdtZW50YXRpb25zOiB6ZXJvIGFsbCBwaXhlbHMgPCAyNDAnKQpjb25mdXNpb25fbWF0cml4KHN1YmplY3RfbGV2ZWwsICdmaWx0ZXJfMjQwJykKYGBgCgpgYGB7cn0KY29tcHV0ZV9hdXByKHN1YmplY3RfbGV2ZWwkZmlsdGVyXzI0MF9iaW5hcml6ZWQsIHN1YmplY3RfbGV2ZWwkbGFiZWwsICdQUjogU2VnbWVudGF0aW9uczogemVybyBhbGwgcGl4ZWxzIDwgMjQwIGFuZCBiaW5hcml6ZScpCmNvbXB1dGVfYXVyb2Moc3ViamVjdF9sZXZlbCRmaWx0ZXJfMjQwX2JpbmFyaXplZCwgc3ViamVjdF9sZXZlbCRsYWJlbCwgJ1JPQzogU2VnbWVudGF0aW9uczogemVybyBhbGwgcGl4ZWxzIDwgMjQwIGFuZCBiaW5hcml6ZScpCmNvbmZ1c2lvbl9tYXRyaXgoc3ViamVjdF9sZXZlbCwgJ2ZpbHRlcl8yNDBfYmluYXJpemVkJykKYGBgCgpgYGB7cn0KY29tcHV0ZV9hdXByKHN1YmplY3RfbGV2ZWwkZmlsdGVyXzI0MF9za2VsZXRvbml6ZWQsIHN1YmplY3RfbGV2ZWwkbGFiZWwsICdQUjogU2VnbWVudGF0aW9uczogemVybyBhbGwgcGl4ZWxzIDwgMjQwIGFuZCBza2VsZXRvbml6ZScpCmNvbXB1dGVfYXVyb2Moc3ViamVjdF9sZXZlbCRmaWx0ZXJfMjQwX3NrZWxldG9uaXplZCwgc3ViamVjdF9sZXZlbCRsYWJlbCwgJ1JPQzogU2VnbWVudGF0aW9uczogemVybyBhbGwgcGl4ZWxzIDwgMjQwIGFuZCBza2VsZXRvbml6ZScpCmNvbmZ1c2lvbl9tYXRyaXgoc3ViamVjdF9sZXZlbCwgJ2ZpbHRlcl8yNDBfc2tlbGV0b25pemVkJykKYGBgCgoKCiMjIyBTZWdtZW50YXRpb25zOiB6ZXJvIGFsbCBwaXhlbHMgPCAyNTAKCmBgYHtyfQpjb21wdXRlX2F1cHIoc3ViamVjdF9sZXZlbCRmaWx0ZXJfMjUwLCBzdWJqZWN0X2xldmVsJGxhYmVsLCAnUFI6IFNlZ21lbnRhdGlvbnM6IHplcm8gYWxsIHBpeGVscyA8IDI1MCcpCmNvbXB1dGVfYXVyb2Moc3ViamVjdF9sZXZlbCRmaWx0ZXJfMjUwLCBzdWJqZWN0X2xldmVsJGxhYmVsLCAnUk9DOiBTZWdtZW50YXRpb25zOiB6ZXJvIGFsbCBwaXhlbHMgPCAyNTAnKQpjb25mdXNpb25fbWF0cml4KHN1YmplY3RfbGV2ZWwsICdmaWx0ZXJfMjUwJykKYGBgCgpgYGB7cn0KY29tcHV0ZV9hdXByKHN1YmplY3RfbGV2ZWwkZmlsdGVyXzI1MF9iaW5hcml6ZWQsIHN1YmplY3RfbGV2ZWwkbGFiZWwsICdQUjogU2VnbWVudGF0aW9uczogemVybyBhbGwgcGl4ZWxzIDwgMjUwIGFuZCBiaW5hcml6ZScpCmNvbXB1dGVfYXVyb2Moc3ViamVjdF9sZXZlbCRmaWx0ZXJfMjUwX2JpbmFyaXplZCwgc3ViamVjdF9sZXZlbCRsYWJlbCwgJ1JPQzogU2VnbWVudGF0aW9uczogemVybyBhbGwgcGl4ZWxzIDwgMjUwIGFuZCBiaW5hcml6ZScpCmNvbmZ1c2lvbl9tYXRyaXgoc3ViamVjdF9sZXZlbCwgJ2ZpbHRlcl8yNTBfYmluYXJpemVkJykKYGBgCgpgYGB7cn0KY29tcHV0ZV9hdXByKHN1YmplY3RfbGV2ZWwkZmlsdGVyXzI1MF9za2VsZXRvbml6ZWQsIHN1YmplY3RfbGV2ZWwkbGFiZWwsICdQUjogU2VnbWVudGF0aW9uczogemVybyBhbGwgcGl4ZWxzIDwgMjUwIGFuZCBza2VsZXRvbml6ZScpCmNvbXB1dGVfYXVyb2Moc3ViamVjdF9sZXZlbCRmaWx0ZXJfMjUwX3NrZWxldG9uaXplZCwgc3ViamVjdF9sZXZlbCRsYWJlbCwgJ1JPQzogU2VnbWVudGF0aW9uczogemVybyBhbGwgcGl4ZWxzIDwgMjUwIGFuZCBza2VsZXRvbml6ZScpCmNvbmZ1c2lvbl9tYXRyaXgoc3ViamVjdF9sZXZlbCwgJ2ZpbHRlcl8yNTBfc2tlbGV0b25pemVkJykKYGBgCgoKCiMjIyBTZWdtZW50YXRpb25zOiB6ZXJvIGFsbCBwaXhlbHMgPCAyNTcKCmBgYHtyfQpjb21wdXRlX2F1cHIoc3ViamVjdF9sZXZlbCRmaWx0ZXJfMjU3LCBzdWJqZWN0X2xldmVsJGxhYmVsLCAnUFI6IFNlZ21lbnRhdGlvbnM6IHplcm8gYWxsIHBpeGVscyA8IDI1NycpCmNvbXB1dGVfYXVyb2Moc3ViamVjdF9sZXZlbCRmaWx0ZXJfMjU3LCBzdWJqZWN0X2xldmVsJGxhYmVsLCAnUk9DOiBTZWdtZW50YXRpb25zOiB6ZXJvIGFsbCBwaXhlbHMgPCAyNTcnKQpjb25mdXNpb25fbWF0cml4KHN1YmplY3RfbGV2ZWwsICdmaWx0ZXJfMjU3JykKYGBgCgpgYGB7cn0KY29tcHV0ZV9hdXByKHN1YmplY3RfbGV2ZWwkZmlsdGVyXzI1N19iaW5hcml6ZWQsIHN1YmplY3RfbGV2ZWwkbGFiZWwsICdQUjogU2VnbWVudGF0aW9uczogemVybyBhbGwgcGl4ZWxzIDwgMjU3IGFuZCBiaW5hcml6ZScpCmNvbXB1dGVfYXVyb2Moc3ViamVjdF9sZXZlbCRmaWx0ZXJfMjU3X2JpbmFyaXplZCwgc3ViamVjdF9sZXZlbCRsYWJlbCwgJ1JPQzogU2VnbWVudGF0aW9uczogemVybyBhbGwgcGl4ZWxzIDwgMjU3IGFuZCBiaW5hcml6ZScpCmNvbmZ1c2lvbl9tYXRyaXgoc3ViamVjdF9sZXZlbCwgJ2ZpbHRlcl8yNTdfYmluYXJpemVkJykKYGBgCgpgYGB7cn0KY29tcHV0ZV9hdXByKHN1YmplY3RfbGV2ZWwkZmlsdGVyXzI1N19za2VsZXRvbml6ZWQsIHN1YmplY3RfbGV2ZWwkbGFiZWwsICdQUjogU2VnbWVudGF0aW9uczogemVybyBhbGwgcGl4ZWxzIDwgMjU3IGFuZCBza2VsZXRvbml6ZScpCmNvbXB1dGVfYXVyb2Moc3ViamVjdF9sZXZlbCRmaWx0ZXJfMjU3X3NrZWxldG9uaXplZCwgc3ViamVjdF9sZXZlbCRsYWJlbCwgJ1JPQzogU2VnbWVudGF0aW9uczogemVybyBhbGwgcGl4ZWxzIDwgMjU3IGFuZCBza2VsZXRvbml6ZScpCmNvbmZ1c2lvbl9tYXRyaXgoc3ViamVjdF9sZXZlbCwgJ2ZpbHRlcl8yNTdfc2tlbGV0b25pemVkJykKYGBgCgoKCiMjIyBTZWdtZW50YXRpb25zOiB6ZXJvIGFsbCBwaXhlbHMgPiAxMAoKYGBge3J9CmNvbXB1dGVfYXVwcihzdWJqZWN0X2xldmVsJGZpbHRlcl8xMCwgc3ViamVjdF9sZXZlbCRsYWJlbCwgJ1BSOiBTZWdtZW50YXRpb25zOiB6ZXJvIGFsbCBwaXhlbHMgPCAxMCcpCmNvbXB1dGVfYXVyb2Moc3ViamVjdF9sZXZlbCRmaWx0ZXJfMTAsIHN1YmplY3RfbGV2ZWwkbGFiZWwsICdST0M6IFNlZ21lbnRhdGlvbnM6IHplcm8gYWxsIHBpeGVscyA8IDEwJykKY29uZnVzaW9uX21hdHJpeChzdWJqZWN0X2xldmVsLCAnZmlsdGVyXzEwJykKYGBgCgpgYGB7cn0KY29tcHV0ZV9hdXByKHN1YmplY3RfbGV2ZWwkZmlsdGVyXzEwX2JpbmFyaXplZCwgc3ViamVjdF9sZXZlbCRsYWJlbCwgJ1BSOiBTZWdtZW50YXRpb25zOiB6ZXJvIGFsbCBwaXhlbHMgPCAxMCBhbmQgYmluYXJpemUnKQpjb21wdXRlX2F1cm9jKHN1YmplY3RfbGV2ZWwkZmlsdGVyXzEwX2JpbmFyaXplZCwgc3ViamVjdF9sZXZlbCRsYWJlbCwgJ1JPQzogU2VnbWVudGF0aW9uczogemVybyBhbGwgcGl4ZWxzIDwgMTAgYW5kIGJpbmFyaXplJykKY29uZnVzaW9uX21hdHJpeChzdWJqZWN0X2xldmVsLCAnZmlsdGVyXzEwX2JpbmFyaXplZCcpCmBgYAoKYGBge3J9CmNvbXB1dGVfYXVwcihzdWJqZWN0X2xldmVsJGZpbHRlcl8xMF9za2VsZXRvbml6ZWQsIHN1YmplY3RfbGV2ZWwkbGFiZWwsICdQUjogU2VnbWVudGF0aW9uczogemVybyBhbGwgcGl4ZWxzIDwgMTAgYW5kIHNrZWxldG9uaXplJykKY29tcHV0ZV9hdXJvYyhzdWJqZWN0X2xldmVsJGZpbHRlcl8xMF9za2VsZXRvbml6ZWQsIHN1YmplY3RfbGV2ZWwkbGFiZWwsICdST0M6IFNlZ21lbnRhdGlvbnM6IHplcm8gYWxsIHBpeGVscyA8IDEwIGFuZCBza2VsZXRvbml6ZScpCmNvbmZ1c2lvbl9tYXRyaXgoc3ViamVjdF9sZXZlbCwgJ2ZpbHRlcl8xMF9za2VsZXRvbml6ZWQnKQpgYGAKCgoKIyMjIFNlZ21lbnRhdGlvbnM6IHplcm8gYWxsIHBpeGVscyA8IDc1IGFuZCA+IDE1MAoKYGBge3J9CmNvbXB1dGVfYXVwcihzdWJqZWN0X2xldmVsJGZpbHRlcl83NSwgc3ViamVjdF9sZXZlbCRsYWJlbCwgJ1BSOiBTZWdtZW50YXRpb25zOiB6ZXJvIGFsbCBwaXhlbHMgPCA3NSBhbmQgPiAxNTAnKQpjb21wdXRlX2F1cm9jKHN1YmplY3RfbGV2ZWwkZmlsdGVyXzc1LCBzdWJqZWN0X2xldmVsJGxhYmVsLCAnUk9DOiBTZWdtZW50YXRpb25zOiB6ZXJvIGFsbCBwaXhlbHMgPCA3NSBhbmQgPiAxNTAnKQpjb25mdXNpb25fbWF0cml4KHN1YmplY3RfbGV2ZWwsICdmaWx0ZXJfNzUnKQpgYGAKCmBgYHtyfQpjb21wdXRlX2F1cHIoc3ViamVjdF9sZXZlbCRmaWx0ZXJfNzVfYmluYXJpemVkLCBzdWJqZWN0X2xldmVsJGxhYmVsLCAnUFI6IFNlZ21lbnRhdGlvbnM6IHplcm8gYWxsIHBpeGVscyA8IDc1IGFuZCA+IDE1MCBhbmQgYmluYXJpemUnKQpjb21wdXRlX2F1cm9jKHN1YmplY3RfbGV2ZWwkZmlsdGVyXzc1X2JpbmFyaXplZCwgc3ViamVjdF9sZXZlbCRsYWJlbCwgJ1JPQzogU2VnbWVudGF0aW9uczogemVybyBhbGwgcGl4ZWxzIDwgNzUgYW5kID4gMTUwIGFuZCBiaW5hcml6ZScpCmNvbmZ1c2lvbl9tYXRyaXgoc3ViamVjdF9sZXZlbCwgJ2ZpbHRlcl83NV9iaW5hcml6ZWQnKQpgYGAKCmBgYHtyfQpjb21wdXRlX2F1cHIoc3ViamVjdF9sZXZlbCRmaWx0ZXJfNzVfc2tlbGV0b25pemVkLCBzdWJqZWN0X2xldmVsJGxhYmVsLCAnUFI6IFNlZ21lbnRhdGlvbnM6IHplcm8gYWxsIHBpeGVscyA8IDc1IGFuZCA+IDE1MCBhbmQgc2tlbGV0b25pemUnKQpjb21wdXRlX2F1cm9jKHN1YmplY3RfbGV2ZWwkZmlsdGVyXzc1X3NrZWxldG9uaXplZCwgc3ViamVjdF9sZXZlbCRsYWJlbCwgJ1JPQzogU2VnbWVudGF0aW9uczogemVybyBhbGwgcGl4ZWxzIDwgNzUgYW5kID4gMTUwIGFuZCBza2VsZXRvbml6ZScpCmNvbmZ1c2lvbl9tYXRyaXgoc3ViamVjdF9sZXZlbCwgJ2ZpbHRlcl83NV9za2VsZXRvbml6ZWQnKQpgYGAK